youtube_dl/extractor/common.py

   1 from __future__ import unicode_literals
   2
   3 import base64
   4 import datetime
   5 import hashlib
   6 import json
   7 import netrc
   8 import os
   9 import re
  10 import socket
  11 import sys
  12 import time
  13 import xml.etree.ElementTree
  14
  15 from ..compat import (
  16     compat_cookiejar,
  17     compat_cookies,
  18     compat_HTTPError,
  19     compat_http_client,
  20     compat_urllib_error,
  21     compat_urllib_parse_urlparse,
  22     compat_urllib_request,
  23     compat_urlparse,
  24     compat_str,
  25 )
  26 from ..utils import (
  27     NO_DEFAULT,
  28     age_restricted,
  29     bug_reports_message,
  30     clean_html,
  31     compiled_regex_type,
  32     determine_ext,
  33     ExtractorError,
  34     fix_xml_ampersands,
  35     float_or_none,
  36     int_or_none,
  37     RegexNotFoundError,
  38     sanitize_filename,
  39     unescapeHTML,
  40 )
  41
  42
  43 class InfoExtractor(object):
  44     """Information Extractor class.
  45
  46     Information extractors are the classes that, given a URL, extract
  47     information about the video (or videos) the URL refers to. This
  48     information includes the real video URL, the video title, author and
  49     others. The information is stored in a dictionary which is then
  50     passed to the YoutubeDL. The YoutubeDL processes this
  51     information possibly downloading the video to the file system, among
  52     other possible outcomes.
  53
  54     The type field determines the type of the result.
  55     By far the most common value (and the default if _type is missing) is
  56     "video", which indicates a single video.
  57
  58     For a video, the dictionaries must include the following fields:
  59
  60     id:             Video identifier.
  61     title:          Video title, unescaped.
  62
  63     Additionally, it must contain either a formats entry or a url one:
  64
  65     formats:        A list of dictionaries for each format available, ordered
  66                     from worst to best quality.
  67
  68                     Potential fields:
  69                     * url        Mandatory. The URL of the video file
  70                     * ext        Will be calculated from URL if missing
  71                     * format     A human-readable description of the format
  72                                  ("mp4 container with h264/opus").
  73                                  Calculated from the format_id, width, height.
  74                                  and format_note fields if missing.
  75                     * format_id  A short description of the format
  76                                  ("mp4_h264_opus" or "19").
  77                                 Technically optional, but strongly recommended.
  78                     * format_note Additional info about the format
  79                                  ("3D" or "DASH video")
  80                     * width      Width of the video, if known
  81                     * height     Height of the video, if known
  82                     * resolution Textual description of width and height
  83                     * tbr        Average bitrate of audio and video in KBit/s
  84                     * abr        Average audio bitrate in KBit/s
  85                     * acodec     Name of the audio codec in use
  86                     * asr        Audio sampling rate in Hertz
  87                     * vbr        Average video bitrate in KBit/s
  88                     * fps        Frame rate
  89                     * vcodec     Name of the video codec in use
  90                     * container  Name of the container format
  91                     * filesize   The number of bytes, if known in advance
  92                     * filesize_approx  An estimate for the number of bytes
  93                     * player_url SWF Player URL (used for rtmpdump).
  94                     * protocol   The protocol that will be used for the actual
  95                                  download, lower-case.
  96                                  "http", "https", "rtsp", "rtmp", "rtmpe",
  97                                  "m3u8", or "m3u8_native".
  98                     * preference Order number of this format. If this field is
  99                                  present and not None, the formats get sorted
 100                                  by this field, regardless of all other values.
 101                                  -1 for default (order by other properties),
 102                                  -2 or smaller for less than default.
 103                                  < -1000 to hide the format (if there is
 104                                     another one which is strictly better)
 105                     * language_preference  Is this in the correct requested
 106                                  language?
 107                                  10 if it's what the URL is about,
 108                                  -1 for default (don't know),
 109                                  -10 otherwise, other values reserved for now.
 110                     * quality    Order number of the video quality of this
 111                                  format, irrespective of the file format.
 112                                  -1 for default (order by other properties),
 113                                  -2 or smaller for less than default.
 114                     * source_preference  Order number for this video source
 115                                   (quality takes higher priority)
 116                                  -1 for default (order by other properties),
 117                                  -2 or smaller for less than default.
 118                     * http_headers  A dictionary of additional HTTP headers
 119                                  to add to the request.
 120                     * stretched_ratio  If given and not 1, indicates that the
 121                                  video's pixels are not square.
 122                                  width : height ratio as float.
 123                     * no_resume  The server does not support resuming the
 124                                  (HTTP or RTMP) download. Boolean.
 125
 126     url:            Final video URL.
 127     ext:            Video filename extension.
 128     format:         The video format, defaults to ext (used for --get-format)
 129     player_url:     SWF Player URL (used for rtmpdump).
 130
 131     The following fields are optional:
 132
 133     alt_title:      A secondary title of the video.
 134     display_id      An alternative identifier for the video, not necessarily
 135                     unique, but available before title. Typically, id is
 136                     something like "4234987", title "Dancing naked mole rats",
 137                     and display_id "dancing-naked-mole-rats"
 138     thumbnails:     A list of dictionaries, with the following entries:
 139                         * "id" (optional, string) - Thumbnail format ID
 140                         * "url"
 141                         * "preference" (optional, int) - quality of the image
 142                         * "width" (optional, int)
 143                         * "height" (optional, int)
 144                         * "resolution" (optional, string "{width}x{height"},
 145                                         deprecated)
 146     thumbnail:      Full URL to a video thumbnail image.
 147     description:    Full video description.
 148     uploader:       Full name of the video uploader.
 149     creator:        The main artist who created the video.
 150     timestamp:      UNIX timestamp of the moment the video became available.
 151     upload_date:    Video upload date (YYYYMMDD).
 152                     If not explicitly set, calculated from timestamp.
 153     uploader_id:    Nickname or id of the video uploader.
 154     location:       Physical location where the video was filmed.
 155     subtitles:      The available subtitles as a dictionary in the format
 156                     {language: subformats}. "subformats" is a list sorted from
 157                     lower to higher preference, each element is a dictionary
 158                     with the "ext" entry and one of:
 159                         * "data": The subtitles file contents
 160                         * "url": A URL pointing to the subtitles file
 161     automatic_captions: Like 'subtitles', used by the YoutubeIE for
 162                     automatically generated captions
 163     duration:       Length of the video in seconds, as an integer.
 164     view_count:     How many users have watched the video on the platform.
 165     like_count:     Number of positive ratings of the video
 166     dislike_count:  Number of negative ratings of the video
 167     average_rating: Average rating give by users, the scale used depends on the webpage
 168     comment_count:  Number of comments on the video
 169     comments:       A list of comments, each with one or more of the following
 170                     properties (all but one of text or html optional):
 171                         * "author" - human-readable name of the comment author
 172                         * "author_id" - user ID of the comment author
 173                         * "id" - Comment ID
 174                         * "html" - Comment as HTML
 175                         * "text" - Plain text of the comment
 176                         * "timestamp" - UNIX timestamp of comment
 177                         * "parent" - ID of the comment this one is replying to.
 178                                      Set to "root" to indicate that this is a
 179                                      comment to the original video.
 180     age_limit:      Age restriction for the video, as an integer (years)
 181     webpage_url:    The URL to the video webpage, if given to youtube-dl it
 182                     should allow to get the same result again. (It will be set
 183                     by YoutubeDL if it's missing)
 184     categories:     A list of categories that the video falls in, for example
 185                     ["Sports", "Berlin"]
 186     tags:           A list of tags assigned to the video, e.g. ["sweden", "pop music"]
 187     is_live:        True, False, or None (=unknown). Whether this video is a
 188                     live stream that goes on instead of a fixed-length video.
 189     start_time:     Time in seconds where the reproduction should start, as
 190                     specified in the URL.
 191     end_time:       Time in seconds where the reproduction should end, as
 192                     specified in the URL.
 193
 194     Unless mentioned otherwise, the fields should be Unicode strings.
 195
 196     Unless mentioned otherwise, None is equivalent to absence of information.
 197
 198
 199     _type "playlist" indicates multiple videos.
 200     There must be a key "entries", which is a list, an iterable, or a PagedList
 201     object, each element of which is a valid dictionary by this specification.
 202
 203     Additionally, playlists can have "title" and "id" attributes with the same
 204     semantics as videos (see above).
 205
 206
 207     _type "multi_video" indicates that there are multiple videos that
 208     form a single show, for examples multiple acts of an opera or TV episode.
 209     It must have an entries key like a playlist and contain all the keys
 210     required for a video at the same time.
 211
 212
 213     _type "url" indicates that the video must be extracted from another
 214     location, possibly by a different extractor. Its only required key is:
 215     "url" - the next URL to extract.
 216     The key "ie_key" can be set to the class name (minus the trailing "IE",
 217     e.g. "Youtube") if the extractor class is known in advance.
 218     Additionally, the dictionary may have any properties of the resolved entity
 219     known in advance, for example "title" if the title of the referred video is
 220     known ahead of time.
 221
 222
 223     _type "url_transparent" entities have the same specification as "url", but
 224     indicate that the given additional information is more precise than the one
 225     associated with the resolved URL.
 226     This is useful when a site employs a video service that hosts the video and
 227     its technical metadata, but that video service does not embed a useful
 228     title, description etc.
 229
 230
 231     Subclasses of this one should re-define the _real_initialize() and
 232     _real_extract() methods and define a _VALID_URL regexp.
 233     Probably, they should also be added to the list of extractors.
 234
 235     Finally, the _WORKING attribute should be set to False for broken IEs
 236     in order to warn the users and skip the tests.
 237     """
 238
 239     _ready = False
 240     _downloader = None
 241     _WORKING = True
 242
 243     def __init__(self, downloader=None):
 244         """Constructor. Receives an optional downloader."""
 245         self._ready = False
 246         self.set_downloader(downloader)
 247
 248     @classmethod
 249     def suitable(cls, url):
 250         """Receives a URL and returns True if suitable for this IE."""
 251
 252         # This does not use has/getattr intentionally - we want to know whether
 253         # we have cached the regexp for *this* class, whereas getattr would also
 254         # match the superclass
 255         if '_VALID_URL_RE' not in cls.__dict__:
 256             cls._VALID_URL_RE = re.compile(cls._VALID_URL)
 257         return cls._VALID_URL_RE.match(url) is not None
 258
 259     @classmethod
 260     def _match_id(cls, url):
 261         if '_VALID_URL_RE' not in cls.__dict__:
 262             cls._VALID_URL_RE = re.compile(cls._VALID_URL)
 263         m = cls._VALID_URL_RE.match(url)
 264         assert m
 265         return m.group('id')
 266
 267     @classmethod
 268     def working(cls):
 269         """Getter method for _WORKING."""
 270         return cls._WORKING
 271
 272     def initialize(self):
 273         """Initializes an instance (authentication, etc)."""
 274         if not self._ready:
 275             self._real_initialize()
 276             self._ready = True
 277
 278     def extract(self, url):
 279         """Extracts URL information and returns it in list of dicts."""
 280         try:
 281             self.initialize()
 282             return self._real_extract(url)
 283         except ExtractorError:
 284             raise
 285         except compat_http_client.IncompleteRead as e:
 286             raise ExtractorError('A network error has occured.', cause=e, expected=True)
 287         except (KeyError, StopIteration) as e:
 288             raise ExtractorError('An extractor error has occured.', cause=e)
 289
 290     def set_downloader(self, downloader):
 291         """Sets the downloader for this IE."""
 292         self._downloader = downloader
 293
 294     def _real_initialize(self):
 295         """Real initialization process. Redefine in subclasses."""
 296         pass
 297
 298     def _real_extract(self, url):
 299         """Real extraction process. Redefine in subclasses."""
 300         pass
 301
 302     @classmethod
 303     def ie_key(cls):
 304         """A string for getting the InfoExtractor with get_info_extractor"""
 305         return cls.__name__[:-2]
 306
 307     @property
 308     def IE_NAME(self):
 309         return type(self).__name__[:-2]
 310
 311     def _request_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True):
 312         """ Returns the response handle """
 313         if note is None:
 314             self.report_download_webpage(video_id)
 315         elif note is not False:
 316             if video_id is None:
 317                 self.to_screen('%s' % (note,))
 318             else:
 319                 self.to_screen('%s: %s' % (video_id, note))
 320         try:
 321             return self._downloader.urlopen(url_or_request)
 322         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
 323             if errnote is False:
 324                 return False
 325             if errnote is None:
 326                 errnote = 'Unable to download webpage'
 327             errmsg = '%s: %s' % (errnote, compat_str(err))
 328             if fatal:
 329                 raise ExtractorError(errmsg, sys.exc_info()[2], cause=err)
 330             else:
 331                 self._downloader.report_warning(errmsg)
 332                 return False
 333
 334     def _download_webpage_handle(self, url_or_request, video_id, note=None, errnote=None, fatal=True, encoding=None):
 335         """ Returns a tuple (page content as string, URL handle) """
 336         # Strip hashes from the URL (#1038)
 337         if isinstance(url_or_request, (compat_str, str)):
 338             url_or_request = url_or_request.partition('#')[0]
 339
 340         urlh = self._request_webpage(url_or_request, video_id, note, errnote, fatal)
 341         if urlh is False:
 342             assert not fatal
 343             return False
 344         content = self._webpage_read_content(urlh, url_or_request, video_id, note, errnote, fatal, encoding=encoding)
 345         return (content, urlh)
 346
 347     @staticmethod
 348     def _guess_encoding_from_content(content_type, webpage_bytes):
 349         m = re.match(r'[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+\s*;\s*charset=(.+)', content_type)
 350         if m:
 351             encoding = m.group(1)
 352         else:
 353             m = re.search(br'<meta[^>]+charset=[\'"]?([^\'")]+)[ /\'">]',
 354                           webpage_bytes[:1024])
 355             if m:
 356                 encoding = m.group(1).decode('ascii')
 357             elif webpage_bytes.startswith(b'\xff\xfe'):
 358                 encoding = 'utf-16'
 359             else:
 360                 encoding = 'utf-8'
 361
 362         return encoding
 363
 364     def _webpage_read_content(self, urlh, url_or_request, video_id, note=None, errnote=None, fatal=True, prefix=None, encoding=None):
 365         content_type = urlh.headers.get('Content-Type', '')
 366         webpage_bytes = urlh.read()
 367         if prefix is not None:
 368             webpage_bytes = prefix + webpage_bytes
 369         if not encoding:
 370             encoding = self._guess_encoding_from_content(content_type, webpage_bytes)
 371         if self._downloader.params.get('dump_intermediate_pages', False):
 372             try:
 373                 url = url_or_request.get_full_url()
 374             except AttributeError:
 375                 url = url_or_request
 376             self.to_screen('Dumping request to ' + url)
 377             dump = base64.b64encode(webpage_bytes).decode('ascii')
 378             self._downloader.to_screen(dump)
 379         if self._downloader.params.get('write_pages', False):
 380             try:
 381                 url = url_or_request.get_full_url()
 382             except AttributeError:
 383                 url = url_or_request
 384             basen = '%s_%s' % (video_id, url)
 385             if len(basen) > 240:
 386                 h = '___' + hashlib.md5(basen.encode('utf-8')).hexdigest()
 387                 basen = basen[:240 - len(h)] + h
 388             raw_filename = basen + '.dump'
 389             filename = sanitize_filename(raw_filename, restricted=True)
 390             self.to_screen('Saving request to ' + filename)
 391             # Working around MAX_PATH limitation on Windows (see
 392             # http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx)
 393             if os.name == 'nt':
 394                 absfilepath = os.path.abspath(filename)
 395                 if len(absfilepath) > 259:
 396                     filename = '\\\\?\\' + absfilepath
 397             with open(filename, 'wb') as outf:
 398                 outf.write(webpage_bytes)
 399
 400         try:
 401             content = webpage_bytes.decode(encoding, 'replace')
 402         except LookupError:
 403             content = webpage_bytes.decode('utf-8', 'replace')
 404
 405         if ('<title>Access to this site is blocked</title>' in content and
 406                 'Websense' in content[:512]):
 407             msg = 'Access to this webpage has been blocked by Websense filtering software in your network.'
 408             blocked_iframe = self._html_search_regex(
 409                 r'<iframe src="([^"]+)"', content,
 410                 'Websense information URL', default=None)
 411             if blocked_iframe:
 412                 msg += ' Visit %s for more details' % blocked_iframe
 413             raise ExtractorError(msg, expected=True)
 414         if '<title>The URL you requested has been blocked</title>' in content[:512]:
 415             msg = (
 416                 'Access to this webpage has been blocked by Indian censorship. '
 417                 'Use a VPN or proxy server (with --proxy) to route around it.')
 418             block_msg = self._html_search_regex(
 419                 r'</h1><p>(.*?)</p>',
 420                 content, 'block message', default=None)
 421             if block_msg:
 422                 msg += ' (Message: "%s")' % block_msg.replace('\n', ' ')
 423             raise ExtractorError(msg, expected=True)
 424
 425         return content
 426
 427     def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, tries=1, timeout=5, encoding=None):
 428         """ Returns the data of the page as a string """
 429         success = False
 430         try_count = 0
 431         while success is False:
 432             try:
 433                 res = self._download_webpage_handle(url_or_request, video_id, note, errnote, fatal, encoding=encoding)
 434                 success = True
 435             except compat_http_client.IncompleteRead as e:
 436                 try_count += 1
 437                 if try_count >= tries:
 438                     raise e
 439                 self._sleep(timeout, video_id)
 440         if res is False:
 441             return res
 442         else:
 443             content, _ = res
 444             return content
 445
 446     def _download_xml(self, url_or_request, video_id,
 447                       note='Downloading XML', errnote='Unable to download XML',
 448                       transform_source=None, fatal=True, encoding=None):
 449         """Return the xml as an xml.etree.ElementTree.Element"""
 450         xml_string = self._download_webpage(
 451             url_or_request, video_id, note, errnote, fatal=fatal, encoding=encoding)
 452         if xml_string is False:
 453             return xml_string
 454         if transform_source:
 455             xml_string = transform_source(xml_string)
 456         return xml.etree.ElementTree.fromstring(xml_string.encode('utf-8'))
 457
 458     def _download_json(self, url_or_request, video_id,
 459                        note='Downloading JSON metadata',
 460                        errnote='Unable to download JSON metadata',
 461                        transform_source=None,
 462                        fatal=True, encoding=None):
 463         json_string = self._download_webpage(
 464             url_or_request, video_id, note, errnote, fatal=fatal,
 465             encoding=encoding)
 466         if (not fatal) and json_string is False:
 467             return None
 468         return self._parse_json(
 469             json_string, video_id, transform_source=transform_source, fatal=fatal)
 470
 471     def _parse_json(self, json_string, video_id, transform_source=None, fatal=True):
 472         if transform_source:
 473             json_string = transform_source(json_string)
 474         try:
 475             return json.loads(json_string)
 476         except ValueError as ve:
 477             errmsg = '%s: Failed to parse JSON ' % video_id
 478             if fatal:
 479                 raise ExtractorError(errmsg, cause=ve)
 480             else:
 481                 self.report_warning(errmsg + str(ve))
 482
 483     def report_warning(self, msg, video_id=None):
 484         idstr = '' if video_id is None else '%s: ' % video_id
 485         self._downloader.report_warning(
 486             '[%s] %s%s' % (self.IE_NAME, idstr, msg))
 487
 488     def to_screen(self, msg):
 489         """Print msg to screen, prefixing it with '[ie_name]'"""
 490         self._downloader.to_screen('[%s] %s' % (self.IE_NAME, msg))
 491
 492     def report_extraction(self, id_or_name):
 493         """Report information extraction."""
 494         self.to_screen('%s: Extracting information' % id_or_name)
 495
 496     def report_download_webpage(self, video_id):
 497         """Report webpage download."""
 498         self.to_screen('%s: Downloading webpage' % video_id)
 499
 500     def report_age_confirmation(self):
 501         """Report attempt to confirm age."""
 502         self.to_screen('Confirming age')
 503
 504     def report_login(self):
 505         """Report attempt to log in."""
 506         self.to_screen('Logging in')
 507
 508     # Methods for following #608
 509     @staticmethod
 510     def url_result(url, ie=None, video_id=None, video_title=None):
 511         """Returns a URL that points to a page that should be processed"""
 512         # TODO: ie should be the class used for getting the info
 513         video_info = {'_type': 'url',
 514                       'url': url,
 515                       'ie_key': ie}
 516         if video_id is not None:
 517             video_info['id'] = video_id
 518         if video_title is not None:
 519             video_info['title'] = video_title
 520         return video_info
 521
 522     @staticmethod
 523     def playlist_result(entries, playlist_id=None, playlist_title=None, playlist_description=None):
 524         """Returns a playlist"""
 525         video_info = {'_type': 'playlist',
 526                       'entries': entries}
 527         if playlist_id:
 528             video_info['id'] = playlist_id
 529         if playlist_title:
 530             video_info['title'] = playlist_title
 531         if playlist_description:
 532             video_info['description'] = playlist_description
 533         return video_info
 534
 535     def _search_regex(self, pattern, string, name, default=NO_DEFAULT, fatal=True, flags=0, group=None):
 536         """
 537         Perform a regex search on the given string, using a single or a list of
 538         patterns returning the first matching group.
 539         In case of failure return a default value or raise a WARNING or a
 540         RegexNotFoundError, depending on fatal, specifying the field name.
 541         """
 542         if isinstance(pattern, (str, compat_str, compiled_regex_type)):
 543             mobj = re.search(pattern, string, flags)
 544         else:
 545             for p in pattern:
 546                 mobj = re.search(p, string, flags)
 547                 if mobj:
 548                     break
 549
 550         if not self._downloader.params.get('no_color') and os.name != 'nt' and sys.stderr.isatty():
 551             _name = '\033[0;34m%s\033[0m' % name
 552         else:
 553             _name = name
 554
 555         if mobj:
 556             if group is None:
 557                 # return the first matching group
 558                 return next(g for g in mobj.groups() if g is not None)
 559             else:
 560                 return mobj.group(group)
 561         elif default is not NO_DEFAULT:
 562             return default
 563         elif fatal:
 564             raise RegexNotFoundError('Unable to extract %s' % _name)
 565         else:
 566             self._downloader.report_warning('unable to extract %s' % _name + bug_reports_message())
 567             return None
 568
 569     def _html_search_regex(self, pattern, string, name, default=NO_DEFAULT, fatal=True, flags=0, group=None):
 570         """
 571         Like _search_regex, but strips HTML tags and unescapes entities.
 572         """
 573         res = self._search_regex(pattern, string, name, default, fatal, flags, group)
 574         if res:
 575             return clean_html(res).strip()
 576         else:
 577             return res
 578
 579     def _get_login_info(self):
 580         """
 581         Get the login info as (username, password)
 582         It will look in the netrc file using the _NETRC_MACHINE value
 583         If there's no info available, return (None, None)
 584         """
 585         if self._downloader is None:
 586             return (None, None)
 587
 588         username = None
 589         password = None
 590         downloader_params = self._downloader.params
 591
 592         # Attempt to use provided username and password or .netrc data
 593         if downloader_params.get('username', None) is not None:
 594             username = downloader_params['username']
 595             password = downloader_params['password']
 596         elif downloader_params.get('usenetrc', False):
 597             try:
 598                 info = netrc.netrc().authenticators(self._NETRC_MACHINE)
 599                 if info is not None:
 600                     username = info[0]
 601                     password = info[2]
 602                 else:
 603                     raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
 604             except (IOError, netrc.NetrcParseError) as err:
 605                 self._downloader.report_warning('parsing .netrc: %s' % compat_str(err))
 606
 607         return (username, password)
 608
 609     def _get_tfa_info(self):
 610         """
 611         Get the two-factor authentication info
 612         TODO - asking the user will be required for sms/phone verify
 613         currently just uses the command line option
 614         If there's no info available, return None
 615         """
 616         if self._downloader is None:
 617             return None
 618         downloader_params = self._downloader.params
 619
 620         if downloader_params.get('twofactor', None) is not None:
 621             return downloader_params['twofactor']
 622
 623         return None
 624
 625     # Helper functions for extracting OpenGraph info
 626     @staticmethod
 627     def _og_regexes(prop):
 628         content_re = r'content=(?:"([^>]+?)"|\'([^>]+?)\')'
 629         property_re = r'(?:name|property)=[\'"]og:%s[\'"]' % re.escape(prop)
 630         template = r'<meta[^>]+?%s[^>]+?%s'
 631         return [
 632             template % (property_re, content_re),
 633             template % (content_re, property_re),
 634         ]
 635
 636     @staticmethod
 637     def _meta_regex(prop):
 638         return r'''(?isx)<meta
 639                     (?=[^>]+(?:itemprop|name|property)=(["\']?)%s\1)
 640                     [^>]+?content=(["\'])(?P<content>.*?)\2''' % re.escape(prop)
 641
 642     def _og_search_property(self, prop, html, name=None, **kargs):
 643         if name is None:
 644             name = 'OpenGraph %s' % prop
 645         escaped = self._search_regex(self._og_regexes(prop), html, name, flags=re.DOTALL, **kargs)
 646         if escaped is None:
 647             return None
 648         return unescapeHTML(escaped)
 649
 650     def _og_search_thumbnail(self, html, **kargs):
 651         return self._og_search_property('image', html, 'thumbnail URL', fatal=False, **kargs)
 652
 653     def _og_search_description(self, html, **kargs):
 654         return self._og_search_property('description', html, fatal=False, **kargs)
 655
 656     def _og_search_title(self, html, **kargs):
 657         return self._og_search_property('title', html, **kargs)
 658
 659     def _og_search_video_url(self, html, name='video url', secure=True, **kargs):
 660         regexes = self._og_regexes('video') + self._og_regexes('video:url')
 661         if secure:
 662             regexes = self._og_regexes('video:secure_url') + regexes
 663         return self._html_search_regex(regexes, html, name, **kargs)
 664
 665     def _og_search_url(self, html, **kargs):
 666         return self._og_search_property('url', html, **kargs)
 667
 668     def _html_search_meta(self, name, html, display_name=None, fatal=False, **kwargs):
 669         if display_name is None:
 670             display_name = name
 671         return self._html_search_regex(
 672             self._meta_regex(name),
 673             html, display_name, fatal=fatal, group='content', **kwargs)
 674
 675     def _dc_search_uploader(self, html):
 676         return self._html_search_meta('dc.creator', html, 'uploader')
 677
 678     def _rta_search(self, html):
 679         # See http://www.rtalabel.org/index.php?content=howtofaq#single
 680         if re.search(r'(?ix)<meta\s+name="rating"\s+'
 681                      r'     content="RTA-5042-1996-1400-1577-RTA"',
 682                      html):
 683             return 18
 684         return 0
 685
 686     def _media_rating_search(self, html):
 687         # See http://www.tjg-designs.com/WP/metadata-code-examples-adding-metadata-to-your-web-pages/
 688         rating = self._html_search_meta('rating', html)
 689
 690         if not rating:
 691             return None
 692
 693         RATING_TABLE = {
 694             'safe for kids': 0,
 695             'general': 8,
 696             '14 years': 14,
 697             'mature': 17,
 698             'restricted': 19,
 699         }
 700         return RATING_TABLE.get(rating.lower(), None)
 701
 702     def _family_friendly_search(self, html):
 703         # See http://schema.org/VideoObject
 704         family_friendly = self._html_search_meta('isFamilyFriendly', html)
 705
 706         if not family_friendly:
 707             return None
 708
 709         RATING_TABLE = {
 710             '1': 0,
 711             'true': 0,
 712             '0': 18,
 713             'false': 18,
 714         }
 715         return RATING_TABLE.get(family_friendly.lower(), None)
 716
 717     def _twitter_search_player(self, html):
 718         return self._html_search_meta('twitter:player', html,
 719                                       'twitter card player')
 720
 721     @staticmethod
 722     def _hidden_inputs(html):
 723         return dict([
 724             (input.group('name'), input.group('value')) for input in re.finditer(
 725                 r'''(?x)
 726                     <input\s+
 727                         type=(?P<q_hidden>["\'])hidden(?P=q_hidden)\s+
 728                         name=(?P<q_name>["\'])(?P<name>.+?)(?P=q_name)\s+
 729                         (?:id=(?P<q_id>["\']).+?(?P=q_id)\s+)?
 730                         value=(?P<q_value>["\'])(?P<value>.*?)(?P=q_value)
 731                 ''', html)
 732         ])
 733
 734     def _form_hidden_inputs(self, form_id, html):
 735         form = self._search_regex(
 736             r'(?s)<form[^>]+?id=(["\'])%s\1[^>]*>(?P<form>.+?)</form>' % form_id,
 737             html, '%s form' % form_id, group='form')
 738         return self._hidden_inputs(form)
 739
 740     def _sort_formats(self, formats, field_preference=None):
 741         if not formats:
 742             raise ExtractorError('No video formats found')
 743
 744         def _formats_key(f):
 745             # TODO remove the following workaround
 746             from ..utils import determine_ext
 747             if not f.get('ext') and 'url' in f:
 748                 f['ext'] = determine_ext(f['url'])
 749
 750             if isinstance(field_preference, (list, tuple)):
 751                 return tuple(f.get(field) if f.get(field) is not None else -1 for field in field_preference)
 752
 753             preference = f.get('preference')
 754             if preference is None:
 755                 proto = f.get('protocol')
 756                 if proto is None:
 757                     proto = compat_urllib_parse_urlparse(f.get('url', '')).scheme
 758
 759                 preference = 0 if proto in ['http', 'https'] else -0.1
 760                 if f.get('ext') in ['f4f', 'f4m']:  # Not yet supported
 761                     preference -= 0.5
 762
 763             if f.get('vcodec') == 'none':  # audio only
 764                 if self._downloader.params.get('prefer_free_formats'):
 765                     ORDER = ['aac', 'mp3', 'm4a', 'webm', 'ogg', 'opus']
 766                 else:
 767                     ORDER = ['webm', 'opus', 'ogg', 'mp3', 'aac', 'm4a']
 768                 ext_preference = 0
 769                 try:
 770                     audio_ext_preference = ORDER.index(f['ext'])
 771                 except ValueError:
 772                     audio_ext_preference = -1
 773             else:
 774                 if self._downloader.params.get('prefer_free_formats'):
 775                     ORDER = ['flv', 'mp4', 'webm']
 776                 else:
 777                     ORDER = ['webm', 'flv', 'mp4']
 778                 try:
 779                     ext_preference = ORDER.index(f['ext'])
 780                 except ValueError:
 781                     ext_preference = -1
 782                 audio_ext_preference = 0
 783
 784             return (
 785                 preference,
 786                 f.get('language_preference') if f.get('language_preference') is not None else -1,
 787                 f.get('quality') if f.get('quality') is not None else -1,
 788                 f.get('tbr') if f.get('tbr') is not None else -1,
 789                 f.get('filesize') if f.get('filesize') is not None else -1,
 790                 f.get('vbr') if f.get('vbr') is not None else -1,
 791                 f.get('height') if f.get('height') is not None else -1,
 792                 f.get('width') if f.get('width') is not None else -1,
 793                 ext_preference,
 794                 f.get('abr') if f.get('abr') is not None else -1,
 795                 audio_ext_preference,
 796                 f.get('fps') if f.get('fps') is not None else -1,
 797                 f.get('filesize_approx') if f.get('filesize_approx') is not None else -1,
 798                 f.get('source_preference') if f.get('source_preference') is not None else -1,
 799                 f.get('format_id') if f.get('format_id') is not None else '',
 800             )
 801         formats.sort(key=_formats_key)
 802
 803     def _check_formats(self, formats, video_id):
 804         if formats:
 805             formats[:] = filter(
 806                 lambda f: self._is_valid_url(
 807                     f['url'], video_id,
 808                     item='%s video format' % f.get('format_id') if f.get('format_id') else 'video'),
 809                 formats)
 810
 811     def _is_valid_url(self, url, video_id, item='video'):
 812         url = self._proto_relative_url(url, scheme='http:')
 813         # For now assume non HTTP(S) URLs always valid
 814         if not (url.startswith('http://') or url.startswith('https://')):
 815             return True
 816         try:
 817             self._request_webpage(url, video_id, 'Checking %s URL' % item)
 818             return True
 819         except ExtractorError as e:
 820             if isinstance(e.cause, compat_HTTPError):
 821                 self.to_screen(
 822                     '%s: %s URL is invalid, skipping' % (video_id, item))
 823                 return False
 824             raise
 825
 826     def http_scheme(self):
 827         """ Either "http:" or "https:", depending on the user's preferences """
 828         return (
 829             'http:'
 830             if self._downloader.params.get('prefer_insecure', False)
 831             else 'https:')
 832
 833     def _proto_relative_url(self, url, scheme=None):
 834         if url is None:
 835             return url
 836         if url.startswith('//'):
 837             if scheme is None:
 838                 scheme = self.http_scheme()
 839             return scheme + url
 840         else:
 841             return url
 842
 843     def _sleep(self, timeout, video_id, msg_template=None):
 844         if msg_template is None:
 845             msg_template = '%(video_id)s: Waiting for %(timeout)s seconds'
 846         msg = msg_template % {'video_id': video_id, 'timeout': timeout}
 847         self.to_screen(msg)
 848         time.sleep(timeout)
 849
 850     def _extract_f4m_formats(self, manifest_url, video_id, preference=None, f4m_id=None,
 851                              transform_source=lambda s: fix_xml_ampersands(s).strip()):
 852         manifest = self._download_xml(
 853             manifest_url, video_id, 'Downloading f4m manifest',
 854             'Unable to download f4m manifest',
 855             # Some manifests may be malformed, e.g. prosiebensat1 generated manifests
 856             # (see https://github.com/rg3/youtube-dl/issues/6215#issuecomment-121704244)
 857             transform_source=transform_source)
 858
 859         # currently youtube-dl cannot decode the playerVerificationChallenge as Akamai uses Adobe Alchemy
 860         akamai_pv = manifest.find('{http://ns.adobe.com/f4m/1.0}pv-2.0')
 861         if akamai_pv is not None and ';' in akamai_pv.text:
 862             playerVerificationChallenge = akamai_pv.text.split(';')[0]
 863             if playerVerificationChallenge.strip() != '':
 864                 return []
 865
 866         formats = []
 867         manifest_version = '1.0'
 868         media_nodes = manifest.findall('{http://ns.adobe.com/f4m/1.0}media')
 869         if not media_nodes:
 870             manifest_version = '2.0'
 871             media_nodes = manifest.findall('{http://ns.adobe.com/f4m/2.0}media')
 872         for i, media_el in enumerate(media_nodes):
 873             if manifest_version == '2.0':
 874                 media_url = media_el.attrib.get('href') or media_el.attrib.get('url')
 875                 if not media_url:
 876                     continue
 877                 manifest_url = (
 878                     media_url if media_url.startswith('http://') or media_url.startswith('https://')
 879                     else ('/'.join(manifest_url.split('/')[:-1]) + '/' + media_url))
 880                 # If media_url is itself a f4m manifest do the recursive extraction
 881                 # since bitrates in parent manifest (this one) and media_url manifest
 882                 # may differ leading to inability to resolve the format by requested
 883                 # bitrate in f4m downloader
 884                 if determine_ext(manifest_url) == 'f4m':
 885                     formats.extend(self._extract_f4m_formats(manifest_url, video_id, preference, f4m_id))
 886                     continue
 887             tbr = int_or_none(media_el.attrib.get('bitrate'))
 888             formats.append({
 889                 'format_id': '-'.join(filter(None, [f4m_id, compat_str(i if tbr is None else tbr)])),
 890                 'url': manifest_url,
 891                 'ext': 'flv',
 892                 'tbr': tbr,
 893                 'width': int_or_none(media_el.attrib.get('width')),
 894                 'height': int_or_none(media_el.attrib.get('height')),
 895                 'preference': preference,
 896             })
 897         self._sort_formats(formats)
 898
 899         return formats
 900
 901     def _extract_m3u8_formats(self, m3u8_url, video_id, ext=None,
 902                               entry_protocol='m3u8', preference=None,
 903                               m3u8_id=None, note=None, errnote=None,
 904                               fatal=True):
 905
 906         formats = [{
 907             'format_id': '-'.join(filter(None, [m3u8_id, 'meta'])),
 908             'url': m3u8_url,
 909             'ext': ext,
 910             'protocol': 'm3u8',
 911             'preference': preference - 1 if preference else -1,
 912             'resolution': 'multiple',
 913             'format_note': 'Quality selection URL',
 914         }]
 915
 916         format_url = lambda u: (
 917             u
 918             if re.match(r'^https?://', u)
 919             else compat_urlparse.urljoin(m3u8_url, u))
 920
 921         m3u8_doc = self._download_webpage(
 922             m3u8_url, video_id,
 923             note=note or 'Downloading m3u8 information',
 924             errnote=errnote or 'Failed to download m3u8 information',
 925             fatal=fatal)
 926         if m3u8_doc is False:
 927             return m3u8_doc
 928         last_info = None
 929         last_media = None
 930         kv_rex = re.compile(
 931             r'(?P<key>[a-zA-Z_-]+)=(?P<val>"[^"]+"|[^",]+)(?:,|$)')
 932         for line in m3u8_doc.splitlines():
 933             if line.startswith('#EXT-X-STREAM-INF:'):
 934                 last_info = {}
 935                 for m in kv_rex.finditer(line):
 936                     v = m.group('val')
 937                     if v.startswith('"'):
 938                         v = v[1:-1]
 939                     last_info[m.group('key')] = v
 940             elif line.startswith('#EXT-X-MEDIA:'):
 941                 last_media = {}
 942                 for m in kv_rex.finditer(line):
 943                     v = m.group('val')
 944                     if v.startswith('"'):
 945                         v = v[1:-1]
 946                     last_media[m.group('key')] = v
 947             elif line.startswith('#') or not line.strip():
 948                 continue
 949             else:
 950                 if last_info is None:
 951                     formats.append({'url': format_url(line)})
 952                     continue
 953                 tbr = int_or_none(last_info.get('BANDWIDTH'), scale=1000)
 954                 format_id = []
 955                 if m3u8_id:
 956                     format_id.append(m3u8_id)
 957                 last_media_name = last_media.get('NAME') if last_media and last_media.get('TYPE') != 'SUBTITLES' else None
 958                 format_id.append(last_media_name if last_media_name else '%d' % (tbr if tbr else len(formats)))
 959                 f = {
 960                     'format_id': '-'.join(format_id),
 961                     'url': format_url(line.strip()),
 962                     'tbr': tbr,
 963                     'ext': ext,
 964                     'protocol': entry_protocol,
 965                     'preference': preference,
 966                 }
 967                 codecs = last_info.get('CODECS')
 968                 if codecs:
 969                     # TODO: looks like video codec is not always necessarily goes first
 970                     va_codecs = codecs.split(',')
 971                     if va_codecs[0]:
 972                         f['vcodec'] = va_codecs[0].partition('.')[0]
 973                     if len(va_codecs) > 1 and va_codecs[1]:
 974                         f['acodec'] = va_codecs[1].partition('.')[0]
 975                 resolution = last_info.get('RESOLUTION')
 976                 if resolution:
 977                     width_str, height_str = resolution.split('x')
 978                     f['width'] = int(width_str)
 979                     f['height'] = int(height_str)
 980                 if last_media is not None:
 981                     f['m3u8_media'] = last_media
 982                     last_media = None
 983                 formats.append(f)
 984                 last_info = {}
 985         self._sort_formats(formats)
 986         return formats
 987
 988     # TODO: improve extraction
 989     def _extract_smil_formats(self, smil_url, video_id, fatal=True):
 990         smil = self._download_xml(
 991             smil_url, video_id, 'Downloading SMIL file',
 992             'Unable to download SMIL file', fatal=fatal)
 993         if smil is False:
 994             assert not fatal
 995             return []
 996
 997         base = smil.find('./head/meta').get('base')
 998
 999         formats = []
1000         rtmp_count = 0
1001         if smil.findall('./body/seq/video'):
1002             video = smil.findall('./body/seq/video')[0]
1003             fmts, rtmp_count = self._parse_smil_video(video, video_id, base, rtmp_count)
1004             formats.extend(fmts)
1005         else:
1006             for video in smil.findall('./body/switch/video'):
1007                 fmts, rtmp_count = self._parse_smil_video(video, video_id, base, rtmp_count)
1008                 formats.extend(fmts)
1009
1010         self._sort_formats(formats)
1011
1012         return formats
1013
1014     def _parse_smil_video(self, video, video_id, base, rtmp_count):
1015         src = video.get('src')
1016         if not src:
1017             return [], rtmp_count
1018         bitrate = int_or_none(video.get('system-bitrate') or video.get('systemBitrate'), 1000)
1019         width = int_or_none(video.get('width'))
1020         height = int_or_none(video.get('height'))
1021         proto = video.get('proto')
1022         if not proto:
1023             if base:
1024                 if base.startswith('rtmp'):
1025                     proto = 'rtmp'
1026                 elif base.startswith('http'):
1027                     proto = 'http'
1028         ext = video.get('ext')
1029         if proto == 'm3u8':
1030             return self._extract_m3u8_formats(src, video_id, ext), rtmp_count
1031         elif proto == 'rtmp':
1032             rtmp_count += 1
1033             streamer = video.get('streamer') or base
1034             return ([{
1035                 'url': streamer,
1036                 'play_path': src,
1037                 'ext': 'flv',
1038                 'format_id': 'rtmp-%d' % (rtmp_count if bitrate is None else bitrate),
1039                 'tbr': bitrate,
1040                 'width': width,
1041                 'height': height,
1042             }], rtmp_count)
1043         elif proto.startswith('http'):
1044             return ([{
1045                 'url': base + src,
1046                 'ext': ext or 'flv',
1047                 'tbr': bitrate,
1048                 'width': width,
1049                 'height': height,
1050             }], rtmp_count)
1051
1052     def _live_title(self, name):
1053         """ Generate the title for a live video """
1054         now = datetime.datetime.now()
1055         now_str = now.strftime("%Y-%m-%d %H:%M")
1056         return name + ' ' + now_str
1057
1058     def _int(self, v, name, fatal=False, **kwargs):
1059         res = int_or_none(v, **kwargs)
1060         if 'get_attr' in kwargs:
1061             print(getattr(v, kwargs['get_attr']))
1062         if res is None:
1063             msg = 'Failed to extract %s: Could not parse value %r' % (name, v)
1064             if fatal:
1065                 raise ExtractorError(msg)
1066             else:
1067                 self._downloader.report_warning(msg)
1068         return res
1069
1070     def _float(self, v, name, fatal=False, **kwargs):
1071         res = float_or_none(v, **kwargs)
1072         if res is None:
1073             msg = 'Failed to extract %s: Could not parse value %r' % (name, v)
1074             if fatal:
1075                 raise ExtractorError(msg)
1076             else:
1077                 self._downloader.report_warning(msg)
1078         return res
1079
1080     def _set_cookie(self, domain, name, value, expire_time=None):
1081         cookie = compat_cookiejar.Cookie(
1082             0, name, value, None, None, domain, None,
1083             None, '/', True, False, expire_time, '', None, None, None)
1084         self._downloader.cookiejar.set_cookie(cookie)
1085
1086     def _get_cookies(self, url):
1087         """ Return a compat_cookies.SimpleCookie with the cookies for the url """
1088         req = compat_urllib_request.Request(url)
1089         self._downloader.cookiejar.add_cookie_header(req)
1090         return compat_cookies.SimpleCookie(req.get_header('Cookie'))
1091
1092     def get_testcases(self, include_onlymatching=False):
1093         t = getattr(self, '_TEST', None)
1094         if t:
1095             assert not hasattr(self, '_TESTS'), \
1096                 '%s has _TEST and _TESTS' % type(self).__name__
1097             tests = [t]
1098         else:
1099             tests = getattr(self, '_TESTS', [])
1100         for t in tests:
1101             if not include_onlymatching and t.get('only_matching', False):
1102                 continue
1103             t['name'] = type(self).__name__[:-len('IE')]
1104             yield t
1105
1106     def is_suitable(self, age_limit):
1107         """ Test whether the extractor is generally suitable for the given
1108         age limit (i.e. pornographic sites are not, all others usually are) """
1109
1110         any_restricted = False
1111         for tc in self.get_testcases(include_onlymatching=False):
1112             if 'playlist' in tc:
1113                 tc = tc['playlist'][0]
1114             is_restricted = age_restricted(
1115                 tc.get('info_dict', {}).get('age_limit'), age_limit)
1116             if not is_restricted:
1117                 return True
1118             any_restricted = any_restricted or is_restricted
1119         return not any_restricted
1120
1121     def extract_subtitles(self, *args, **kwargs):
1122         if (self._downloader.params.get('writesubtitles', False) or
1123                 self._downloader.params.get('listsubtitles')):
1124             return self._get_subtitles(*args, **kwargs)
1125         return {}
1126
1127     def _get_subtitles(self, *args, **kwargs):
1128         raise NotImplementedError("This method must be implemented by subclasses")
1129
1130     def extract_automatic_captions(self, *args, **kwargs):
1131         if (self._downloader.params.get('writeautomaticsub', False) or
1132                 self._downloader.params.get('listsubtitles')):
1133             return self._get_automatic_captions(*args, **kwargs)
1134         return {}
1135
1136     def _get_automatic_captions(self, *args, **kwargs):
1137         raise NotImplementedError("This method must be implemented by subclasses")
1138
1139
1140 class SearchInfoExtractor(InfoExtractor):
1141     """
1142     Base class for paged search queries extractors.
1143     They accept URLs in the format _SEARCH_KEY(|all|[0-9]):{query}
1144     Instances should define _SEARCH_KEY and _MAX_RESULTS.
1145     """
1146
1147     @classmethod
1148     def _make_valid_url(cls):
1149         return r'%s(?P<prefix>|[1-9][0-9]*|all):(?P<query>[\s\S]+)' % cls._SEARCH_KEY
1150
1151     @classmethod
1152     def suitable(cls, url):
1153         return re.match(cls._make_valid_url(), url) is not None
1154
1155     def _real_extract(self, query):
1156         mobj = re.match(self._make_valid_url(), query)
1157         if mobj is None:
1158             raise ExtractorError('Invalid search query "%s"' % query)
1159
1160         prefix = mobj.group('prefix')
1161         query = mobj.group('query')
1162         if prefix == '':
1163             return self._get_n_results(query, 1)
1164         elif prefix == 'all':
1165             return self._get_n_results(query, self._MAX_RESULTS)
1166         else:
1167             n = int(prefix)
1168             if n <= 0:
1169                 raise ExtractorError('invalid download number %s for query "%s"' % (n, query))
1170             elif n > self._MAX_RESULTS:
1171                 self._downloader.report_warning('%s returns max %i results (you requested %i)' % (self._SEARCH_KEY, self._MAX_RESULTS, n))
1172                 n = self._MAX_RESULTS
1173             return self._get_n_results(query, n)
1174
1175     def _get_n_results(self, query, n):
1176         """Get a specified number of results for a query"""
1177         raise NotImplementedError("This method must be implemented by subclasses")
1178
1179     @property
1180     def SEARCH_KEY(self):
1181         return self._SEARCH_KEY