youtube_dl/extractor/generic.py

   1 # encoding: utf-8
   2
   3 from __future__ import unicode_literals
   4
   5 import os
   6 import re
   7 import sys
   8
   9 from .common import InfoExtractor
  10 from .youtube import YoutubeIE
  11 from ..compat import (
  12     compat_etree_fromstring,
  13     compat_urllib_parse_unquote,
  14     compat_urlparse,
  15     compat_xml_parse_error,
  16 )
  17 from ..utils import (
  18     determine_ext,
  19     ExtractorError,
  20     float_or_none,
  21     HEADRequest,
  22     is_html,
  23     orderedSet,
  24     sanitized_Request,
  25     smuggle_url,
  26     unescapeHTML,
  27     unified_strdate,
  28     unsmuggle_url,
  29     UnsupportedError,
  30     url_basename,
  31     xpath_text,
  32 )
  33 from .brightcove import (
  34     BrightcoveLegacyIE,
  35     BrightcoveNewIE,
  36 )
  37 from .nbc import NBCSportsVPlayerIE
  38 from .ooyala import OoyalaIE
  39 from .rutv import RUTVIE
  40 from .tvc import TVCIE
  41 from .sportbox import SportBoxEmbedIE
  42 from .smotri import SmotriIE
  43 from .myvi import MyviIE
  44 from .condenast import CondeNastIE
  45 from .udn import UDNEmbedIE
  46 from .senateisvp import SenateISVPIE
  47 from .svt import SVTIE
  48 from .pornhub import PornHubIE
  49 from .xhamster import XHamsterEmbedIE
  50 from .tnaflix import TNAFlixNetworkEmbedIE
  51 from .vimeo import VimeoIE
  52 from .dailymotion import (
  53     DailymotionIE,
  54     DailymotionCloudIE,
  55 )
  56 from .onionstudios import OnionStudiosIE
  57 from .viewlift import ViewLiftEmbedIE
  58 from .screenwavemedia import ScreenwaveMediaIE
  59 from .mtv import MTVServicesEmbeddedIE
  60 from .pladform import PladformIE
  61 from .videomore import VideomoreIE
  62 from .googledrive import GoogleDriveIE
  63 from .jwplatform import JWPlatformIE
  64 from .digiteka import DigitekaIE
  65 from .arkena import ArkenaIE
  66 from .instagram import InstagramIE
  67 from .liveleak import LiveLeakIE
  68 from .threeqsdn import ThreeQSDNIE
  69 from .theplatform import ThePlatformIE
  70 from .vessel import VesselIE
  71 from .kaltura import KalturaIE
  72 from .eagleplatform import EaglePlatformIE
  73 from .facebook import FacebookIE
  74
  75
  76 class GenericIE(InfoExtractor):
  77     IE_DESC = 'Generic downloader that works on some sites'
  78     _VALID_URL = r'.*'
  79     IE_NAME = 'generic'
  80     _TESTS = [
  81         # Direct link to a video
  82         {
  83             'url': 'http://media.w3.org/2010/05/sintel/trailer.mp4',
  84             'md5': '67d406c2bcb6af27fa886f31aa934bbe',
  85             'info_dict': {
  86                 'id': 'trailer',
  87                 'ext': 'mp4',
  88                 'title': 'trailer',
  89                 'upload_date': '20100513',
  90             }
  91         },
  92         # Direct link to media delivered compressed (until Accept-Encoding is *)
  93         {
  94             'url': 'http://calimero.tk/muzik/FictionJunction-Parallel_Hearts.flac',
  95             'md5': '128c42e68b13950268b648275386fc74',
  96             'info_dict': {
  97                 'id': 'FictionJunction-Parallel_Hearts',
  98                 'ext': 'flac',
  99                 'title': 'FictionJunction-Parallel_Hearts',
 100                 'upload_date': '20140522',
 101             },
 102             'expected_warnings': [
 103                 'URL could be a direct video link, returning it as such.'
 104             ]
 105         },
 106         # Direct download with broken HEAD
 107         {
 108             'url': 'http://ai-radio.org:8000/radio.opus',
 109             'info_dict': {
 110                 'id': 'radio',
 111                 'ext': 'opus',
 112                 'title': 'radio',
 113             },
 114             'params': {
 115                 'skip_download': True,  # infinite live stream
 116             },
 117             'expected_warnings': [
 118                 r'501.*Not Implemented',
 119                 r'400.*Bad Request',
 120             ],
 121         },
 122         # Direct link with incorrect MIME type
 123         {
 124             'url': 'http://ftp.nluug.nl/video/nluug/2014-11-20_nj14/zaal-2/5_Lennart_Poettering_-_Systemd.webm',
 125             'md5': '4ccbebe5f36706d85221f204d7eb5913',
 126             'info_dict': {
 127                 'url': 'http://ftp.nluug.nl/video/nluug/2014-11-20_nj14/zaal-2/5_Lennart_Poettering_-_Systemd.webm',
 128                 'id': '5_Lennart_Poettering_-_Systemd',
 129                 'ext': 'webm',
 130                 'title': '5_Lennart_Poettering_-_Systemd',
 131                 'upload_date': '20141120',
 132             },
 133             'expected_warnings': [
 134                 'URL could be a direct video link, returning it as such.'
 135             ]
 136         },
 137         # RSS feed
 138         {
 139             'url': 'http://phihag.de/2014/youtube-dl/rss2.xml',
 140             'info_dict': {
 141                 'id': 'http://phihag.de/2014/youtube-dl/rss2.xml',
 142                 'title': 'Zero Punctuation',
 143                 'description': 're:.*groundbreaking video review series.*'
 144             },
 145             'playlist_mincount': 11,
 146         },
 147         # RSS feed with enclosure
 148         {
 149             'url': 'http://podcastfeeds.nbcnews.com/audio/podcast/MSNBC-MADDOW-NETCAST-M4V.xml',
 150             'info_dict': {
 151                 'id': 'pdv_maddow_netcast_m4v-02-27-2015-201624',
 152                 'ext': 'm4v',
 153                 'upload_date': '20150228',
 154                 'title': 'pdv_maddow_netcast_m4v-02-27-2015-201624',
 155             }
 156         },
 157         # SMIL from http://videolectures.net/promogram_igor_mekjavic_eng
 158         {
 159             'url': 'http://videolectures.net/promogram_igor_mekjavic_eng/video/1/smil.xml',
 160             'info_dict': {
 161                 'id': 'smil',
 162                 'ext': 'mp4',
 163                 'title': 'Automatics, robotics and biocybernetics',
 164                 'description': 'md5:815fc1deb6b3a2bff99de2d5325be482',
 165                 'upload_date': '20130627',
 166                 'formats': 'mincount:16',
 167                 'subtitles': 'mincount:1',
 168             },
 169             'params': {
 170                 'force_generic_extractor': True,
 171                 'skip_download': True,
 172             },
 173         },
 174         # SMIL from http://www1.wdr.de/mediathek/video/livestream/index.html
 175         {
 176             'url': 'http://metafilegenerator.de/WDR/WDR_FS/hds/hds.smil',
 177             'info_dict': {
 178                 'id': 'hds',
 179                 'ext': 'flv',
 180                 'title': 'hds',
 181                 'formats': 'mincount:1',
 182             },
 183             'params': {
 184                 'skip_download': True,
 185             },
 186         },
 187         # SMIL from https://www.restudy.dk/video/play/id/1637
 188         {
 189             'url': 'https://www.restudy.dk/awsmedia/SmilDirectory/video_1637.xml',
 190             'info_dict': {
 191                 'id': 'video_1637',
 192                 'ext': 'flv',
 193                 'title': 'video_1637',
 194                 'formats': 'mincount:3',
 195             },
 196             'params': {
 197                 'skip_download': True,
 198             },
 199         },
 200         # SMIL from http://adventure.howstuffworks.com/5266-cool-jobs-iditarod-musher-video.htm
 201         {
 202             'url': 'http://services.media.howstuffworks.com/videos/450221/smil-service.smil',
 203             'info_dict': {
 204                 'id': 'smil-service',
 205                 'ext': 'flv',
 206                 'title': 'smil-service',
 207                 'formats': 'mincount:1',
 208             },
 209             'params': {
 210                 'skip_download': True,
 211             },
 212         },
 213         # SMIL from http://new.livestream.com/CoheedandCambria/WebsterHall/videos/4719370
 214         {
 215             'url': 'http://api.new.livestream.com/accounts/1570303/events/1585861/videos/4719370.smil',
 216             'info_dict': {
 217                 'id': '4719370',
 218                 'ext': 'mp4',
 219                 'title': '571de1fd-47bc-48db-abf9-238872a58d1f',
 220                 'formats': 'mincount:3',
 221             },
 222             'params': {
 223                 'skip_download': True,
 224             },
 225         },
 226         # XSPF playlist from http://www.telegraaf.nl/tv/nieuws/binnenland/24353229/__Tikibad_ontruimd_wegens_brand__.html
 227         {
 228             'url': 'http://www.telegraaf.nl/xml/playlist/2015/8/7/mZlp2ctYIUEB.xspf',
 229             'info_dict': {
 230                 'id': 'mZlp2ctYIUEB',
 231                 'ext': 'mp4',
 232                 'title': 'Tikibad ontruimd wegens brand',
 233                 'description': 'md5:05ca046ff47b931f9b04855015e163a4',
 234                 'thumbnail': 're:^https?://.*\.jpg$',
 235                 'duration': 33,
 236             },
 237             'params': {
 238                 'skip_download': True,
 239             },
 240         },
 241         # MPD from http://dash-mse-test.appspot.com/media.html
 242         {
 243             'url': 'http://yt-dash-mse-test.commondatastorage.googleapis.com/media/car-20120827-manifest.mpd',
 244             'md5': '4b57baab2e30d6eb3a6a09f0ba57ef53',
 245             'info_dict': {
 246                 'id': 'car-20120827-manifest',
 247                 'ext': 'mp4',
 248                 'title': 'car-20120827-manifest',
 249                 'formats': 'mincount:9',
 250                 'upload_date': '20130904',
 251             },
 252             'params': {
 253                 'format': 'bestvideo',
 254             },
 255         },
 256         # m3u8 served with Content-Type: audio/x-mpegURL; charset=utf-8
 257         {
 258             'url': 'http://once.unicornmedia.com/now/master/playlist/bb0b18ba-64f5-4b1b-a29f-0ac252f06b68/77a785f3-5188-4806-b788-0893a61634ed/93677179-2d99-4ef4-9e17-fe70d49abfbf/content.m3u8',
 259             'info_dict': {
 260                 'id': 'content',
 261                 'ext': 'mp4',
 262                 'title': 'content',
 263                 'formats': 'mincount:8',
 264             },
 265             'params': {
 266                 # m3u8 downloads
 267                 'skip_download': True,
 268             }
 269         },
 270         # m3u8 served with Content-Type: text/plain
 271         {
 272             'url': 'http://www.nacentapps.com/m3u8/index.m3u8',
 273             'info_dict': {
 274                 'id': 'index',
 275                 'ext': 'mp4',
 276                 'title': 'index',
 277                 'upload_date': '20140720',
 278                 'formats': 'mincount:11',
 279             },
 280             'params': {
 281                 # m3u8 downloads
 282                 'skip_download': True,
 283             }
 284         },
 285         # google redirect
 286         {
 287             'url': 'http://www.google.com/url?sa=t&rct=j&q=&esrc=s&source=web&cd=1&cad=rja&ved=0CCUQtwIwAA&url=http%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DcmQHVoWB5FY&ei=F-sNU-LLCaXk4QT52ICQBQ&usg=AFQjCNEw4hL29zgOohLXvpJ-Bdh2bils1Q&bvm=bv.61965928,d.bGE',
 288             'info_dict': {
 289                 'id': 'cmQHVoWB5FY',
 290                 'ext': 'mp4',
 291                 'upload_date': '20130224',
 292                 'uploader_id': 'TheVerge',
 293                 'description': 're:^Chris Ziegler takes a look at the\.*',
 294                 'uploader': 'The Verge',
 295                 'title': 'First Firefox OS phones side-by-side',
 296             },
 297             'params': {
 298                 'skip_download': False,
 299             }
 300         },
 301         {
 302             # redirect in Refresh HTTP header
 303             'url': 'https://www.facebook.com/l.php?u=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DpO8h3EaFRdo&h=TAQHsoToz&enc=AZN16h-b6o4Zq9pZkCCdOLNKMN96BbGMNtcFwHSaazus4JHT_MFYkAA-WARTX2kvsCIdlAIyHZjl6d33ILIJU7Jzwk_K3mcenAXoAzBNoZDI_Q7EXGDJnIhrGkLXo_LJ_pAa2Jzbx17UHMd3jAs--6j2zaeto5w9RTn8T_1kKg3fdC5WPX9Dbb18vzH7YFX0eSJmoa6SP114rvlkw6pkS1-T&s=1',
 304             'info_dict': {
 305                 'id': 'pO8h3EaFRdo',
 306                 'ext': 'mp4',
 307                 'title': 'Tripeo Boiler Room x Dekmantel Festival DJ Set',
 308                 'description': 'md5:6294cc1af09c4049e0652b51a2df10d5',
 309                 'upload_date': '20150917',
 310                 'uploader_id': 'brtvofficial',
 311                 'uploader': 'Boiler Room',
 312             },
 313             'params': {
 314                 'skip_download': False,
 315             },
 316         },
 317         {
 318             'url': 'http://www.hodiho.fr/2013/02/regis-plante-sa-jeep.html',
 319             'md5': '85b90ccc9d73b4acd9138d3af4c27f89',
 320             'info_dict': {
 321                 'id': '13601338388002',
 322                 'ext': 'mp4',
 323                 'uploader': 'www.hodiho.fr',
 324                 'title': 'R\u00e9gis plante sa Jeep',
 325             }
 326         },
 327         # bandcamp page with custom domain
 328         {
 329             'add_ie': ['Bandcamp'],
 330             'url': 'http://bronyrock.com/track/the-pony-mash',
 331             'info_dict': {
 332                 'id': '3235767654',
 333                 'ext': 'mp3',
 334                 'title': 'The Pony Mash',
 335                 'uploader': 'M_Pallante',
 336             },
 337             'skip': 'There is a limit of 200 free downloads / month for the test song',
 338         },
 339         # embedded brightcove video
 340         # it also tests brightcove videos that need to set the 'Referer' in the
 341         # http requests
 342         {
 343             'add_ie': ['BrightcoveLegacy'],
 344             'url': 'http://www.bfmtv.com/video/bfmbusiness/cours-bourse/cours-bourse-l-analyse-technique-154522/',
 345             'info_dict': {
 346                 'id': '2765128793001',
 347                 'ext': 'mp4',
 348                 'title': 'Le cours de bourse : l’analyse technique',
 349                 'description': 'md5:7e9ad046e968cb2d1114004aba466fd9',
 350                 'uploader': 'BFM BUSINESS',
 351             },
 352             'params': {
 353                 'skip_download': True,
 354             },
 355         },
 356         {
 357             # https://github.com/rg3/youtube-dl/issues/2253
 358             'url': 'http://bcove.me/i6nfkrc3',
 359             'md5': '0ba9446db037002366bab3b3eb30c88c',
 360             'info_dict': {
 361                 'id': '3101154703001',
 362                 'ext': 'mp4',
 363                 'title': 'Still no power',
 364                 'uploader': 'thestar.com',
 365                 'description': 'Mississauga resident David Farmer is still out of power as a result of the ice storm a month ago. To keep the house warm, Farmer cuts wood from his property for a wood burning stove downstairs.',
 366             },
 367             'add_ie': ['BrightcoveLegacy'],
 368         },
 369         {
 370             'url': 'http://www.championat.com/video/football/v/87/87499.html',
 371             'md5': 'fb973ecf6e4a78a67453647444222983',
 372             'info_dict': {
 373                 'id': '3414141473001',
 374                 'ext': 'mp4',
 375                 'title': 'Видео. Удаление Дзагоева (ЦСКА)',
 376                 'description': 'Онлайн-трансляция матча ЦСКА - "Волга"',
 377                 'uploader': 'Championat',
 378             },
 379         },
 380         {
 381             # https://github.com/rg3/youtube-dl/issues/3541
 382             'add_ie': ['BrightcoveLegacy'],
 383             'url': 'http://www.kijk.nl/sbs6/leermijvrouwenkennen/videos/jqMiXKAYan2S/aflevering-1',
 384             'info_dict': {
 385                 'id': '3866516442001',
 386                 'ext': 'mp4',
 387                 'title': 'Leer mij vrouwen kennen: Aflevering 1',
 388                 'description': 'Leer mij vrouwen kennen: Aflevering 1',
 389                 'uploader': 'SBS Broadcasting',
 390             },
 391             'skip': 'Restricted to Netherlands',
 392             'params': {
 393                 'skip_download': True,  # m3u8 download
 394             },
 395         },
 396         # ooyala video
 397         {
 398             'url': 'http://www.rollingstone.com/music/videos/norwegian-dj-cashmere-cat-goes-spartan-on-with-me-premiere-20131219',
 399             'md5': '166dd577b433b4d4ebfee10b0824d8ff',
 400             'info_dict': {
 401                 'id': 'BwY2RxaTrTkslxOfcan0UCf0YqyvWysJ',
 402                 'ext': 'mp4',
 403                 'title': '2cc213299525360.mov',  # that's what we get
 404                 'duration': 238.231,
 405             },
 406             'add_ie': ['Ooyala'],
 407         },
 408         {
 409             # ooyala video embedded with http://player.ooyala.com/iframe.js
 410             'url': 'http://www.macrumors.com/2015/07/24/steve-jobs-the-man-in-the-machine-first-trailer/',
 411             'info_dict': {
 412                 'id': 'p0MGJndjoG5SOKqO_hZJuZFPB-Tr5VgB',
 413                 'ext': 'mp4',
 414                 'title': '"Steve Jobs: Man in the Machine" trailer',
 415                 'description': 'The first trailer for the Alex Gibney documentary "Steve Jobs: Man in the Machine."',
 416                 'duration': 135.427,
 417             },
 418             'params': {
 419                 'skip_download': True,
 420             },
 421         },
 422         # embed.ly video
 423         {
 424             'url': 'http://www.tested.com/science/weird/460206-tested-grinding-coffee-2000-frames-second/',
 425             'info_dict': {
 426                 'id': '9ODmcdjQcHQ',
 427                 'ext': 'mp4',
 428                 'title': 'Tested: Grinding Coffee at 2000 Frames Per Second',
 429                 'upload_date': '20140225',
 430                 'description': 'md5:06a40fbf30b220468f1e0957c0f558ff',
 431                 'uploader': 'Tested',
 432                 'uploader_id': 'testedcom',
 433             },
 434             # No need to test YoutubeIE here
 435             'params': {
 436                 'skip_download': True,
 437             },
 438         },
 439         # funnyordie embed
 440         {
 441             'url': 'http://www.theguardian.com/world/2014/mar/11/obama-zach-galifianakis-between-two-ferns',
 442             'info_dict': {
 443                 'id': '18e820ec3f',
 444                 'ext': 'mp4',
 445                 'title': 'Between Two Ferns with Zach Galifianakis: President Barack Obama',
 446                 'description': 'Episode 18: President Barack Obama sits down with Zach Galifianakis for his most memorable interview yet.',
 447             },
 448         },
 449         # RUTV embed
 450         {
 451             'url': 'http://www.rg.ru/2014/03/15/reg-dfo/anklav-anons.html',
 452             'info_dict': {
 453                 'id': '776940',
 454                 'ext': 'mp4',
 455                 'title': 'Охотское море стало целиком российским',
 456                 'description': 'md5:5ed62483b14663e2a95ebbe115eb8f43',
 457             },
 458             'params': {
 459                 # m3u8 download
 460                 'skip_download': True,
 461             },
 462         },
 463         # TVC embed
 464         {
 465             'url': 'http://sch1298sz.mskobr.ru/dou_edu/karamel_ki/filial_galleries/video/iframe_src_http_tvc_ru_video_iframe_id_55304_isplay_false_acc_video_id_channel_brand_id_11_show_episodes_episode_id_32307_frameb/',
 466             'info_dict': {
 467                 'id': '55304',
 468                 'ext': 'mp4',
 469                 'title': 'Дошкольное воспитание',
 470             },
 471         },
 472         # SportBox embed
 473         {
 474             'url': 'http://www.vestifinance.ru/articles/25753',
 475             'info_dict': {
 476                 'id': '25753',
 477                 'title': 'Прямые трансляции с Форума-выставки "Госзаказ-2013"',
 478             },
 479             'playlist': [{
 480                 'info_dict': {
 481                     'id': '370908',
 482                     'title': 'Госзаказ. День 3',
 483                     'ext': 'mp4',
 484                 }
 485             }, {
 486                 'info_dict': {
 487                     'id': '370905',
 488                     'title': 'Госзаказ. День 2',
 489                     'ext': 'mp4',
 490                 }
 491             }, {
 492                 'info_dict': {
 493                     'id': '370902',
 494                     'title': 'Госзаказ. День 1',
 495                     'ext': 'mp4',
 496                 }
 497             }],
 498             'params': {
 499                 # m3u8 download
 500                 'skip_download': True,
 501             },
 502         },
 503         # Myvi.ru embed
 504         {
 505             'url': 'http://www.kinomyvi.tv/news/detail/Pervij-dublirovannij-trejler--Uzhastikov-_nOw1',
 506             'info_dict': {
 507                 'id': 'f4dafcad-ff21-423d-89b5-146cfd89fa1e',
 508                 'ext': 'mp4',
 509                 'title': 'Ужастики, русский трейлер (2015)',
 510                 'thumbnail': 're:^https?://.*\.jpg$',
 511                 'duration': 153,
 512             }
 513         },
 514         # XHamster embed
 515         {
 516             'url': 'http://www.numisc.com/forum/showthread.php?11696-FM15-which-pumiscer-was-this-%28-vid-%29-%28-alfa-as-fuck-srx-%29&s=711f5db534502e22260dec8c5e2d66d8',
 517             'info_dict': {
 518                 'id': 'showthread',
 519                 'title': '[NSFL] [FM15] which pumiscer was this ( vid ) ( alfa as fuck srx )',
 520             },
 521             'playlist_mincount': 7,
 522         },
 523         # Embedded TED video
 524         {
 525             'url': 'http://en.support.wordpress.com/videos/ted-talks/',
 526             'md5': '65fdff94098e4a607385a60c5177c638',
 527             'info_dict': {
 528                 'id': '1969',
 529                 'ext': 'mp4',
 530                 'title': 'Hidden miracles of the natural world',
 531                 'uploader': 'Louie Schwartzberg',
 532                 'description': 'md5:8145d19d320ff3e52f28401f4c4283b9',
 533             }
 534         },
 535         # Embedded Ustream video
 536         {
 537             'url': 'http://www.american.edu/spa/pti/nsa-privacy-janus-2014.cfm',
 538             'md5': '27b99cdb639c9b12a79bca876a073417',
 539             'info_dict': {
 540                 'id': '45734260',
 541                 'ext': 'flv',
 542                 'uploader': 'AU SPA:  The NSA and Privacy',
 543                 'title': 'NSA and Privacy Forum Debate featuring General Hayden and Barton Gellman'
 544             }
 545         },
 546         # nowvideo embed hidden behind percent encoding
 547         {
 548             'url': 'http://www.waoanime.tv/the-super-dimension-fortress-macross-episode-1/',
 549             'md5': '2baf4ddd70f697d94b1c18cf796d5107',
 550             'info_dict': {
 551                 'id': '06e53103ca9aa',
 552                 'ext': 'flv',
 553                 'title': 'Macross Episode 001  Watch Macross Episode 001 onl',
 554                 'description': 'No description',
 555             },
 556         },
 557         # arte embed
 558         {
 559             'url': 'http://www.tv-replay.fr/redirection/20-03-14/x-enius-arte-10753389.html',
 560             'md5': '7653032cbb25bf6c80d80f217055fa43',
 561             'info_dict': {
 562                 'id': '048195-004_PLUS7-F',
 563                 'ext': 'flv',
 564                 'title': 'X:enius',
 565                 'description': 'md5:d5fdf32ef6613cdbfd516ae658abf168',
 566                 'upload_date': '20140320',
 567             },
 568             'params': {
 569                 'skip_download': 'Requires rtmpdump'
 570             }
 571         },
 572         # francetv embed
 573         {
 574             'url': 'http://www.tsprod.com/replay-du-concert-alcaline-de-calogero',
 575             'info_dict': {
 576                 'id': 'EV_30231',
 577                 'ext': 'mp4',
 578                 'title': 'Alcaline, le concert avec Calogero',
 579                 'description': 'md5:61f08036dcc8f47e9cfc33aed08ffaff',
 580                 'upload_date': '20150226',
 581                 'timestamp': 1424989860,
 582                 'duration': 5400,
 583             },
 584             'params': {
 585                 # m3u8 downloads
 586                 'skip_download': True,
 587             },
 588             'expected_warnings': [
 589                 'Forbidden'
 590             ]
 591         },
 592         # Condé Nast embed
 593         {
 594             'url': 'http://www.wired.com/2014/04/honda-asimo/',
 595             'md5': 'ba0dfe966fa007657bd1443ee672db0f',
 596             'info_dict': {
 597                 'id': '53501be369702d3275860000',
 598                 'ext': 'mp4',
 599                 'title': 'Honda’s  New Asimo Robot Is More Human Than Ever',
 600             }
 601         },
 602         # Dailymotion embed
 603         {
 604             'url': 'http://www.spi0n.com/zap-spi0n-com-n216/',
 605             'md5': '441aeeb82eb72c422c7f14ec533999cd',
 606             'info_dict': {
 607                 'id': 'k2mm4bCdJ6CQ2i7c8o2',
 608                 'ext': 'mp4',
 609                 'title': 'Le Zap de Spi0n n°216 - Zapping du Web',
 610                 'description': 'md5:faf028e48a461b8b7fad38f1e104b119',
 611                 'uploader': 'Spi0n',
 612                 'uploader_id': 'xgditw',
 613                 'upload_date': '20140425',
 614                 'timestamp': 1398441542,
 615             },
 616             'add_ie': ['Dailymotion'],
 617         },
 618         # YouTube embed
 619         {
 620             'url': 'http://www.badzine.de/ansicht/datum/2014/06/09/so-funktioniert-die-neue-englische-badminton-liga.html',
 621             'info_dict': {
 622                 'id': 'FXRb4ykk4S0',
 623                 'ext': 'mp4',
 624                 'title': 'The NBL Auction 2014',
 625                 'uploader': 'BADMINTON England',
 626                 'uploader_id': 'BADMINTONEvents',
 627                 'upload_date': '20140603',
 628                 'description': 'md5:9ef128a69f1e262a700ed83edb163a73',
 629             },
 630             'add_ie': ['Youtube'],
 631             'params': {
 632                 'skip_download': True,
 633             }
 634         },
 635         # MTVSercices embed
 636         {
 637             'url': 'http://www.vulture.com/2016/06/new-key-peele-sketches-released.html',
 638             'md5': 'ca1aef97695ef2c1d6973256a57e5252',
 639             'info_dict': {
 640                 'id': '769f7ec0-0692-4d62-9b45-0d88074bffc1',
 641                 'ext': 'mp4',
 642                 'title': 'Key and Peele|October 10, 2012|2|203|Liam Neesons - Uncensored',
 643                 'description': 'Two valets share their love for movie star Liam Neesons.',
 644                 'timestamp': 1349922600,
 645                 'upload_date': '20121011',
 646             },
 647         },
 648         # YouTube embed via <data-embed-url="">
 649         {
 650             'url': 'https://play.google.com/store/apps/details?id=com.gameloft.android.ANMP.GloftA8HM',
 651             'info_dict': {
 652                 'id': '4vAffPZIT44',
 653                 'ext': 'mp4',
 654                 'title': 'Asphalt 8: Airborne - Update - Welcome to Dubai!',
 655                 'uploader': 'Gameloft',
 656                 'uploader_id': 'gameloft',
 657                 'upload_date': '20140828',
 658                 'description': 'md5:c80da9ed3d83ae6d1876c834de03e1c4',
 659             },
 660             'params': {
 661                 'skip_download': True,
 662             }
 663         },
 664         # Camtasia studio
 665         {
 666             'url': 'http://www.ll.mit.edu/workshops/education/videocourses/antennas/lecture1/video/',
 667             'playlist': [{
 668                 'md5': '0c5e352edabf715d762b0ad4e6d9ee67',
 669                 'info_dict': {
 670                     'id': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final',
 671                     'title': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final - video1',
 672                     'ext': 'flv',
 673                     'duration': 2235.90,
 674                 }
 675             }, {
 676                 'md5': '10e4bb3aaca9fd630e273ff92d9f3c63',
 677                 'info_dict': {
 678                     'id': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final_PIP',
 679                     'title': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final - pip',
 680                     'ext': 'flv',
 681                     'duration': 2235.93,
 682                 }
 683             }],
 684             'info_dict': {
 685                 'title': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final',
 686             }
 687         },
 688         # Flowplayer
 689         {
 690             'url': 'http://www.handjobhub.com/video/busty-blonde-siri-tit-fuck-while-wank-6313.html',
 691             'md5': '9d65602bf31c6e20014319c7d07fba27',
 692             'info_dict': {
 693                 'id': '5123ea6d5e5a7',
 694                 'ext': 'mp4',
 695                 'age_limit': 18,
 696                 'uploader': 'www.handjobhub.com',
 697                 'title': 'Busty Blonde Siri Tit Fuck While Wank at HandjobHub.com',
 698             }
 699         },
 700         # Multiple brightcove videos
 701         # https://github.com/rg3/youtube-dl/issues/2283
 702         {
 703             'url': 'http://www.newyorker.com/online/blogs/newsdesk/2014/01/always-never-nuclear-command-and-control.html',
 704             'info_dict': {
 705                 'id': 'always-never',
 706                 'title': 'Always / Never - The New Yorker',
 707             },
 708             'playlist_count': 3,
 709             'params': {
 710                 'extract_flat': False,
 711                 'skip_download': True,
 712             }
 713         },
 714         # MLB embed
 715         {
 716             'url': 'http://umpire-empire.com/index.php/topic/58125-laz-decides-no-thats-low/',
 717             'md5': '96f09a37e44da40dd083e12d9a683327',
 718             'info_dict': {
 719                 'id': '33322633',
 720                 'ext': 'mp4',
 721                 'title': 'Ump changes call to ball',
 722                 'description': 'md5:71c11215384298a172a6dcb4c2e20685',
 723                 'duration': 48,
 724                 'timestamp': 1401537900,
 725                 'upload_date': '20140531',
 726                 'thumbnail': 're:^https?://.*\.jpg$',
 727             },
 728         },
 729         # Wistia embed
 730         {
 731             'url': 'http://study.com/academy/lesson/north-american-exploration-failed-colonies-of-spain-france-england.html#lesson',
 732             'md5': '1953f3a698ab51cfc948ed3992a0b7ff',
 733             'info_dict': {
 734                 'id': '6e2wtrbdaf',
 735                 'ext': 'mov',
 736                 'title': 'paywall_north-american-exploration-failed-colonies-of-spain-france-england',
 737                 'description': 'a Paywall Videos video from Remilon',
 738                 'duration': 644.072,
 739                 'uploader': 'study.com',
 740                 'timestamp': 1459678540,
 741                 'upload_date': '20160403',
 742                 'filesize': 24687186,
 743             },
 744         },
 745         {
 746             'url': 'http://thoughtworks.wistia.com/medias/uxjb0lwrcz',
 747             'md5': 'baf49c2baa8a7de5f3fc145a8506dcd4',
 748             'info_dict': {
 749                 'id': 'uxjb0lwrcz',
 750                 'ext': 'mp4',
 751                 'title': 'Conversation about Hexagonal Rails Part 1',
 752                 'description': 'a Martin Fowler video from ThoughtWorks',
 753                 'duration': 1715.0,
 754                 'uploader': 'thoughtworks.wistia.com',
 755                 'timestamp': 1401832161,
 756                 'upload_date': '20140603',
 757             },
 758         },
 759         # Wistia standard embed (async)
 760         {
 761             'url': 'https://www.getdrip.com/university/brennan-dunn-drip-workshop/',
 762             'info_dict': {
 763                 'id': '807fafadvk',
 764                 'ext': 'mp4',
 765                 'title': 'Drip Brennan Dunn Workshop',
 766                 'description': 'a JV Webinars video from getdrip-1',
 767                 'duration': 4986.95,
 768                 'timestamp': 1463607249,
 769                 'upload_date': '20160518',
 770             },
 771             'params': {
 772                 'skip_download': True,
 773             }
 774         },
 775         # Soundcloud embed
 776         {
 777             'url': 'http://nakedsecurity.sophos.com/2014/10/29/sscc-171-are-you-sure-that-1234-is-a-bad-password-podcast/',
 778             'info_dict': {
 779                 'id': '174391317',
 780                 'ext': 'mp3',
 781                 'description': 'md5:ff867d6b555488ad3c52572bb33d432c',
 782                 'uploader': 'Sophos Security',
 783                 'title': 'Chet Chat 171 - Oct 29, 2014',
 784                 'upload_date': '20141029',
 785             }
 786         },
 787         # Livestream embed
 788         {
 789             'url': 'http://www.esa.int/Our_Activities/Space_Science/Rosetta/Philae_comet_touch-down_webcast',
 790             'info_dict': {
 791                 'id': '67864563',
 792                 'ext': 'flv',
 793                 'upload_date': '20141112',
 794                 'title': 'Rosetta #CometLanding webcast HL 10',
 795             }
 796         },
 797         # Another Livestream embed, without 'new.' in URL
 798         {
 799             'url': 'https://www.freespeech.org/',
 800             'info_dict': {
 801                 'id': '123537347',
 802                 'ext': 'mp4',
 803                 'title': 're:^FSTV [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
 804             },
 805             'params': {
 806                 # Live stream
 807                 'skip_download': True,
 808             },
 809         },
 810         # LazyYT
 811         {
 812             'url': 'http://discourse.ubuntu.com/t/unity-8-desktop-mode-windows-on-mir/1986',
 813             'info_dict': {
 814                 'id': '1986',
 815                 'title': 'Unity 8 desktop-mode windows on Mir! - Ubuntu Discourse',
 816             },
 817             'playlist_mincount': 2,
 818         },
 819         # Cinchcast embed
 820         {
 821             'url': 'http://undergroundwellness.com/podcasts/306-5-steps-to-permanent-gut-healing/',
 822             'info_dict': {
 823                 'id': '7141703',
 824                 'ext': 'mp3',
 825                 'upload_date': '20141126',
 826                 'title': 'Jack Tips: 5 Steps to Permanent Gut Healing',
 827             }
 828         },
 829         # Cinerama player
 830         {
 831             'url': 'http://www.abc.net.au/7.30/content/2015/s4164797.htm',
 832             'info_dict': {
 833                 'id': '730m_DandD_1901_512k',
 834                 'ext': 'mp4',
 835                 'uploader': 'www.abc.net.au',
 836                 'title': 'Game of Thrones with dice - Dungeons and Dragons fantasy role-playing game gets new life - 19/01/2015',
 837             }
 838         },
 839         # embedded viddler video
 840         {
 841             'url': 'http://deadspin.com/i-cant-stop-watching-john-wall-chop-the-nuggets-with-th-1681801597',
 842             'info_dict': {
 843                 'id': '4d03aad9',
 844                 'ext': 'mp4',
 845                 'uploader': 'deadspin',
 846                 'title': 'WALL-TO-GORTAT',
 847                 'timestamp': 1422285291,
 848                 'upload_date': '20150126',
 849             },
 850             'add_ie': ['Viddler'],
 851         },
 852         # Libsyn embed
 853         {
 854             'url': 'http://thedailyshow.cc.com/podcast/episodetwelve',
 855             'info_dict': {
 856                 'id': '3377616',
 857                 'ext': 'mp3',
 858                 'title': "The Daily Show Podcast without Jon Stewart - Episode 12: Bassem Youssef: Egypt's Jon Stewart",
 859                 'description': 'md5:601cb790edd05908957dae8aaa866465',
 860                 'upload_date': '20150220',
 861             },
 862         },
 863         # jwplayer YouTube
 864         {
 865             'url': 'http://media.nationalarchives.gov.uk/index.php/webinar-using-discovery-national-archives-online-catalogue/',
 866             'info_dict': {
 867                 'id': 'Mrj4DVp2zeA',
 868                 'ext': 'mp4',
 869                 'upload_date': '20150212',
 870                 'uploader': 'The National Archives UK',
 871                 'description': 'md5:a236581cd2449dd2df4f93412f3f01c6',
 872                 'uploader_id': 'NationalArchives08',
 873                 'title': 'Webinar: Using Discovery, The National Archives’ online catalogue',
 874             },
 875         },
 876         # rtl.nl embed
 877         {
 878             'url': 'http://www.rtlnieuws.nl/nieuws/buitenland/aanslagen-kopenhagen',
 879             'playlist_mincount': 5,
 880             'info_dict': {
 881                 'id': 'aanslagen-kopenhagen',
 882                 'title': 'Aanslagen Kopenhagen | RTL Nieuws',
 883             }
 884         },
 885         # Zapiks embed
 886         {
 887             'url': 'http://www.skipass.com/news/116090-bon-appetit-s5ep3-baqueira-mi-cor.html',
 888             'info_dict': {
 889                 'id': '118046',
 890                 'ext': 'mp4',
 891                 'title': 'EP3S5 - Bon Appétit - Baqueira Mi Corazon !',
 892             }
 893         },
 894         # Kaltura embed (different embed code)
 895         {
 896             'url': 'http://www.premierchristianradio.com/Shows/Saturday/Unbelievable/Conference-Videos/Os-Guinness-Is-It-Fools-Talk-Unbelievable-Conference-2014',
 897             'info_dict': {
 898                 'id': '1_a52wc67y',
 899                 'ext': 'flv',
 900                 'upload_date': '20150127',
 901                 'uploader_id': 'PremierMedia',
 902                 'timestamp': int,
 903                 'title': 'Os Guinness // Is It Fools Talk? // Unbelievable? Conference 2014',
 904             },
 905         },
 906         # Kaltura embed protected with referrer
 907         {
 908             'url': 'http://www.disney.nl/disney-channel/filmpjes/achter-de-schermen#/videoId/violetta-achter-de-schermen-ruggero',
 909             'info_dict': {
 910                 'id': '1_g4fbemnq',
 911                 'ext': 'mp4',
 912                 'title': 'Violetta - Achter De Schermen - Ruggero',
 913                 'description': 'Achter de schermen met Ruggero',
 914                 'timestamp': 1435133761,
 915                 'upload_date': '20150624',
 916                 'uploader_id': 'echojecka',
 917             },
 918         },
 919         # Kaltura embed with single quotes
 920         {
 921             'url': 'http://fod.infobase.com/p_ViewPlaylist.aspx?AssignmentID=NUN8ZY',
 922             'info_dict': {
 923                 'id': '0_izeg5utt',
 924                 'ext': 'mp4',
 925                 'title': '35871',
 926                 'timestamp': 1355743100,
 927                 'upload_date': '20121217',
 928                 'uploader_id': 'batchUser',
 929             },
 930             'add_ie': ['Kaltura'],
 931         },
 932         {
 933             # Kaltura embedded via quoted entry_id
 934             'url': 'https://www.oreilly.com/ideas/my-cloud-makes-pretty-pictures',
 935             'info_dict': {
 936                 'id': '0_utuok90b',
 937                 'ext': 'mp4',
 938                 'title': '06_matthew_brender_raj_dutt',
 939                 'timestamp': 1466638791,
 940                 'upload_date': '20160622',
 941             },
 942             'add_ie': ['Kaltura'],
 943             'expected_warnings': [
 944                 'Could not send HEAD request'
 945             ],
 946             'params': {
 947                 'skip_download': True,
 948             }
 949         },
 950         # Eagle.Platform embed (generic URL)
 951         {
 952             'url': 'http://lenta.ru/news/2015/03/06/navalny/',
 953             # Not checking MD5 as sometimes the direct HTTP link results in 404 and HLS is used
 954             'info_dict': {
 955                 'id': '227304',
 956                 'ext': 'mp4',
 957                 'title': 'Навальный вышел на свободу',
 958                 'description': 'md5:d97861ac9ae77377f3f20eaf9d04b4f5',
 959                 'thumbnail': 're:^https?://.*\.jpg$',
 960                 'duration': 87,
 961                 'view_count': int,
 962                 'age_limit': 0,
 963             },
 964         },
 965         # ClipYou (Eagle.Platform) embed (custom URL)
 966         {
 967             'url': 'http://muz-tv.ru/play/7129/',
 968             # Not checking MD5 as sometimes the direct HTTP link results in 404 and HLS is used
 969             'info_dict': {
 970                 'id': '12820',
 971                 'ext': 'mp4',
 972                 'title': "'O Sole Mio",
 973                 'thumbnail': 're:^https?://.*\.jpg$',
 974                 'duration': 216,
 975                 'view_count': int,
 976             },
 977         },
 978         # Pladform embed
 979         {
 980             'url': 'http://muz-tv.ru/kinozal/view/7400/',
 981             'info_dict': {
 982                 'id': '100183293',
 983                 'ext': 'mp4',
 984                 'title': 'Тайны перевала Дятлова • 1 серия 2 часть',
 985                 'description': 'Документальный сериал-расследование одной из самых жутких тайн ХХ века',
 986                 'thumbnail': 're:^https?://.*\.jpg$',
 987                 'duration': 694,
 988                 'age_limit': 0,
 989             },
 990         },
 991         # Playwire embed
 992         {
 993             'url': 'http://www.cinemablend.com/new/First-Joe-Dirt-2-Trailer-Teaser-Stupid-Greatness-70874.html',
 994             'info_dict': {
 995                 'id': '3519514',
 996                 'ext': 'mp4',
 997                 'title': 'Joe Dirt 2 Beautiful Loser Teaser Trailer',
 998                 'thumbnail': 're:^https?://.*\.png$',
 999                 'duration': 45.115,
1000             },
1001         },
1002         # 5min embed
1003         {
1004             'url': 'http://techcrunch.com/video/facebook-creates-on-this-day-crunch-report/518726732/',
1005             'md5': '4c6f127a30736b59b3e2c19234ee2bf7',
1006             'info_dict': {
1007                 'id': '518726732',
1008                 'ext': 'mp4',
1009                 'title': 'Facebook Creates "On This Day" | Crunch Report',
1010             },
1011         },
1012         # SVT embed
1013         {
1014             'url': 'http://www.svt.se/sport/ishockey/jagr-tacklar-giroux-under-intervjun',
1015             'info_dict': {
1016                 'id': '2900353',
1017                 'ext': 'flv',
1018                 'title': 'Här trycker Jagr till Giroux (under SVT-intervjun)',
1019                 'duration': 27,
1020                 'age_limit': 0,
1021             },
1022         },
1023         # Crooks and Liars embed
1024         {
1025             'url': 'http://crooksandliars.com/2015/04/fox-friends-says-protecting-atheists',
1026             'info_dict': {
1027                 'id': '8RUoRhRi',
1028                 'ext': 'mp4',
1029                 'title': "Fox & Friends Says Protecting Atheists From Discrimination Is Anti-Christian!",
1030                 'description': 'md5:e1a46ad1650e3a5ec7196d432799127f',
1031                 'timestamp': 1428207000,
1032                 'upload_date': '20150405',
1033                 'uploader': 'Heather',
1034             },
1035         },
1036         # Crooks and Liars external embed
1037         {
1038             'url': 'http://theothermccain.com/2010/02/02/video-proves-that-bill-kristol-has-been-watching-glenn-beck/comment-page-1/',
1039             'info_dict': {
1040                 'id': 'MTE3MjUtMzQ2MzA',
1041                 'ext': 'mp4',
1042                 'title': 'md5:5e3662a81a4014d24c250d76d41a08d5',
1043                 'description': 'md5:9b8e9542d6c3c5de42d6451b7d780cec',
1044                 'timestamp': 1265032391,
1045                 'upload_date': '20100201',
1046                 'uploader': 'Heather',
1047             },
1048         },
1049         # NBC Sports vplayer embed
1050         {
1051             'url': 'http://www.riderfans.com/forum/showthread.php?121827-Freeman&s=e98fa1ea6dc08e886b1678d35212494a',
1052             'info_dict': {
1053                 'id': 'ln7x1qSThw4k',
1054                 'ext': 'flv',
1055                 'title': "PFT Live: New leader in the 'new-look' defense",
1056                 'description': 'md5:65a19b4bbfb3b0c0c5768bed1dfad74e',
1057                 'uploader': 'NBCU-SPORTS',
1058                 'upload_date': '20140107',
1059                 'timestamp': 1389118457,
1060             },
1061         },
1062         # NBC News embed
1063         {
1064             'url': 'http://www.vulture.com/2016/06/letterman-couldnt-care-less-about-late-night.html',
1065             'md5': '1aa589c675898ae6d37a17913cf68d66',
1066             'info_dict': {
1067                 'id': '701714499682',
1068                 'ext': 'mp4',
1069                 'title': 'PREVIEW: On Assignment: David Letterman',
1070                 'description': 'A preview of Tom Brokaw\'s interview with David Letterman as part of the On Assignment series powered by Dateline. Airs Sunday June 12 at 7/6c.',
1071             },
1072         },
1073         # UDN embed
1074         {
1075             'url': 'https://video.udn.com/news/300346',
1076             'md5': 'fd2060e988c326991037b9aff9df21a6',
1077             'info_dict': {
1078                 'id': '300346',
1079                 'ext': 'mp4',
1080                 'title': '中一中男師變性 全校師生力挺',
1081                 'thumbnail': 're:^https?://.*\.jpg$',
1082             },
1083             'params': {
1084                 # m3u8 download
1085                 'skip_download': True,
1086             },
1087         },
1088         # Ooyala embed
1089         {
1090             'url': 'http://www.businessinsider.com/excel-index-match-vlookup-video-how-to-2015-2?IR=T',
1091             'info_dict': {
1092                 'id': '50YnY4czr4ms1vJ7yz3xzq0excz_pUMs',
1093                 'ext': 'mp4',
1094                 'description': 'VIDEO: INDEX/MATCH versus VLOOKUP.',
1095                 'title': 'This is what separates the Excel masters from the wannabes',
1096                 'duration': 191.933,
1097             },
1098             'params': {
1099                 # m3u8 downloads
1100                 'skip_download': True,
1101             }
1102         },
1103         # Brightcove URL in single quotes
1104         {
1105             'url': 'http://www.sportsnet.ca/baseball/mlb/sn-presents-russell-martin-world-citizen/',
1106             'md5': '4ae374f1f8b91c889c4b9203c8c752af',
1107             'info_dict': {
1108                 'id': '4255764656001',
1109                 'ext': 'mp4',
1110                 'title': 'SN Presents: Russell Martin, World Citizen',
1111                 'description': 'To understand why he was the Toronto Blue Jays’ top off-season priority is to appreciate his background and upbringing in Montreal, where he first developed his baseball skills. Written and narrated by Stephen Brunt.',
1112                 'uploader': 'Rogers Sportsnet',
1113                 'uploader_id': '1704050871',
1114                 'upload_date': '20150525',
1115                 'timestamp': 1432570283,
1116             },
1117         },
1118         # Dailymotion Cloud video
1119         {
1120             'url': 'http://replay.publicsenat.fr/vod/le-debat/florent-kolandjian,dominique-cena,axel-decourtye,laurence-abeille,bruno-parmentier/175910',
1121             'md5': 'dcaf23ad0c67a256f4278bce6e0bae38',
1122             'info_dict': {
1123                 'id': 'x2uy8t3',
1124                 'ext': 'mp4',
1125                 'title': 'Sauvons les abeilles ! - Le débat',
1126                 'description': 'md5:d9082128b1c5277987825d684939ca26',
1127                 'thumbnail': 're:^https?://.*\.jpe?g$',
1128                 'timestamp': 1434970506,
1129                 'upload_date': '20150622',
1130                 'uploader': 'Public Sénat',
1131                 'uploader_id': 'xa9gza',
1132             }
1133         },
1134         # OnionStudios embed
1135         {
1136             'url': 'http://www.clickhole.com/video/dont-understand-bitcoin-man-will-mumble-explanatio-2537',
1137             'info_dict': {
1138                 'id': '2855',
1139                 'ext': 'mp4',
1140                 'title': 'Don’t Understand Bitcoin? This Man Will Mumble An Explanation At You',
1141                 'thumbnail': 're:^https?://.*\.jpe?g$',
1142                 'uploader': 'ClickHole',
1143                 'uploader_id': 'clickhole',
1144             }
1145         },
1146         # SnagFilms embed
1147         {
1148             'url': 'http://whilewewatch.blogspot.ru/2012/06/whilewewatch-whilewewatch-gripping.html',
1149             'info_dict': {
1150                 'id': '74849a00-85a9-11e1-9660-123139220831',
1151                 'ext': 'mp4',
1152                 'title': '#whilewewatch',
1153             }
1154         },
1155         # AdobeTVVideo embed
1156         {
1157             'url': 'https://helpx.adobe.com/acrobat/how-to/new-experience-acrobat-dc.html?set=acrobat--get-started--essential-beginners',
1158             'md5': '43662b577c018ad707a63766462b1e87',
1159             'info_dict': {
1160                 'id': '2456',
1161                 'ext': 'mp4',
1162                 'title': 'New experience with Acrobat DC',
1163                 'description': 'New experience with Acrobat DC',
1164                 'duration': 248.667,
1165             },
1166         },
1167         # ScreenwaveMedia embed
1168         {
1169             'url': 'http://www.thecinemasnob.com/the-cinema-snob/a-nightmare-on-elm-street-2-freddys-revenge1',
1170             'md5': '24ace5baba0d35d55c6810b51f34e9e0',
1171             'info_dict': {
1172                 'id': 'cinemasnob-55d26273809dd',
1173                 'ext': 'mp4',
1174                 'title': 'cinemasnob',
1175             },
1176         },
1177         # BrightcoveInPageEmbed embed
1178         {
1179             'url': 'http://www.geekandsundry.com/tabletop-bonus-wils-final-thoughts-on-dread/',
1180             'info_dict': {
1181                 'id': '4238694884001',
1182                 'ext': 'flv',
1183                 'title': 'Tabletop: Dread, Last Thoughts',
1184                 'description': 'Tabletop: Dread, Last Thoughts',
1185                 'duration': 51690,
1186             },
1187         },
1188         # JWPlayer with M3U8
1189         {
1190             'url': 'http://ren.tv/novosti/2015-09-25/sluchaynyy-prohozhiy-poymal-avtougonshchika-v-murmanske-video',
1191             'info_dict': {
1192                 'id': 'playlist',
1193                 'ext': 'mp4',
1194                 'title': 'Случайный прохожий поймал автоугонщика в Мурманске. ВИДЕО | РЕН ТВ',
1195                 'uploader': 'ren.tv',
1196             },
1197             'params': {
1198                 # m3u8 downloads
1199                 'skip_download': True,
1200             }
1201         },
1202         # Brightcove embed, with no valid 'renditions' but valid 'IOSRenditions'
1203         # This video can't be played in browsers if Flash disabled and UA set to iPhone, which is actually a false alarm
1204         {
1205             'url': 'https://dl.dropboxusercontent.com/u/29092637/interview.html',
1206             'info_dict': {
1207                 'id': '4785848093001',
1208                 'ext': 'mp4',
1209                 'title': 'The Cardinal Pell Interview',
1210                 'description': 'Sky News Contributor Andrew Bolt interviews George Pell in Rome, following the Cardinal\'s evidence before the Royal Commission into Child Abuse. ',
1211                 'uploader': 'GlobeCast Australia - GlobeStream',
1212                 'uploader_id': '2733773828001',
1213                 'upload_date': '20160304',
1214                 'timestamp': 1457083087,
1215             },
1216             'params': {
1217                 # m3u8 downloads
1218                 'skip_download': True,
1219             },
1220         },
1221         # Another form of arte.tv embed
1222         {
1223             'url': 'http://www.tv-replay.fr/redirection/09-04-16/arte-reportage-arte-11508975.html',
1224             'md5': '850bfe45417ddf221288c88a0cffe2e2',
1225             'info_dict': {
1226                 'id': '030273-562_PLUS7-F',
1227                 'ext': 'mp4',
1228                 'title': 'ARTE Reportage - Nulle part, en France',
1229                 'description': 'md5:e3a0e8868ed7303ed509b9e3af2b870d',
1230                 'upload_date': '20160409',
1231             },
1232         },
1233         # LiveLeak embed
1234         {
1235             'url': 'http://www.wykop.pl/link/3088787/',
1236             'md5': 'ace83b9ed19b21f68e1b50e844fdf95d',
1237             'info_dict': {
1238                 'id': '874_1459135191',
1239                 'ext': 'mp4',
1240                 'title': 'Man shows poor quality of new apartment building',
1241                 'description': 'The wall is like a sand pile.',
1242                 'uploader': 'Lake8737',
1243             }
1244         },
1245         # Duplicated embedded video URLs
1246         {
1247             'url': 'http://www.hudl.com/athlete/2538180/highlights/149298443',
1248             'info_dict': {
1249                 'id': '149298443_480_16c25b74_2',
1250                 'ext': 'mp4',
1251                 'title': 'vs. Blue Orange Spring Game',
1252                 'uploader': 'www.hudl.com',
1253             },
1254         },
1255         # twitter:player:stream embed
1256         {
1257             'url': 'http://www.rtl.be/info/video/589263.aspx?CategoryID=288',
1258             'info_dict': {
1259                 'id': 'master',
1260                 'ext': 'mp4',
1261                 'title': 'Une nouvelle espèce de dinosaure découverte en Argentine',
1262                 'uploader': 'www.rtl.be',
1263             },
1264             'params': {
1265                 # m3u8 downloads
1266                 'skip_download': True,
1267             },
1268         },
1269         # twitter:player embed
1270         {
1271             'url': 'http://www.theatlantic.com/video/index/484130/what-do-black-holes-sound-like/',
1272             'md5': 'a3e0df96369831de324f0778e126653c',
1273             'info_dict': {
1274                 'id': '4909620399001',
1275                 'ext': 'mp4',
1276                 'title': 'What Do Black Holes Sound Like?',
1277                 'description': 'what do black holes sound like',
1278                 'upload_date': '20160524',
1279                 'uploader_id': '29913724001',
1280                 'timestamp': 1464107587,
1281                 'uploader': 'TheAtlantic',
1282             },
1283             'add_ie': ['BrightcoveLegacy'],
1284         },
1285         # Facebook <iframe> embed
1286         {
1287             'url': 'https://www.hostblogger.de/blog/archives/6181-Auto-jagt-Betonmischer.html',
1288             'md5': 'fbcde74f534176ecb015849146dd3aee',
1289             'info_dict': {
1290                 'id': '599637780109885',
1291                 'ext': 'mp4',
1292                 'title': 'Facebook video #599637780109885',
1293             },
1294         },
1295         # Facebook API embed
1296         {
1297             'url': 'http://www.lothype.com/blue-stars-2016-preview-standstill-full-show/',
1298             'md5': 'a47372ee61b39a7b90287094d447d94e',
1299             'info_dict': {
1300                 'id': '10153467542406923',
1301                 'ext': 'mp4',
1302                 'title': 'Facebook video #10153467542406923',
1303             },
1304         },
1305         # Wordpress "YouTube Video Importer" plugin
1306         {
1307             'url': 'http://www.lothype.com/blue-devils-drumline-stanford-lot-2016/',
1308             'md5': 'd16797741b560b485194eddda8121b48',
1309             'info_dict': {
1310                 'id': 'HNTXWDXV9Is',
1311                 'ext': 'mp4',
1312                 'title': 'Blue Devils Drumline Stanford lot 2016',
1313                 'upload_date': '20160627',
1314                 'uploader_id': 'GENOCIDE8GENERAL10',
1315                 'uploader': 'cylus cyrus',
1316             },
1317         },
1318         {
1319             # video stored on custom kaltura server
1320             'url': 'http://www.expansion.com/multimedia/videos.html?media=EQcM30NHIPv',
1321             'md5': '537617d06e64dfed891fa1593c4b30cc',
1322             'info_dict': {
1323                 'id': '0_1iotm5bh',
1324                 'ext': 'mp4',
1325                 'title': 'Elecciones británicas: 5 lecciones para Rajoy',
1326                 'description': 'md5:435a89d68b9760b92ce67ed227055f16',
1327                 'uploader_id': 'videos.expansion@el-mundo.net',
1328                 'upload_date': '20150429',
1329                 'timestamp': 1430303472,
1330             },
1331             'add_ie': ['Kaltura'],
1332         },
1333         {
1334             # Non-standard Vimeo embed
1335             'url': 'https://openclassrooms.com/courses/understanding-the-web',
1336             'md5': '64d86f1c7d369afd9a78b38cbb88d80a',
1337             'info_dict': {
1338                 'id': '148867247',
1339                 'ext': 'mp4',
1340                 'title': 'Understanding the web - Teaser',
1341                 'description': 'This is "Understanding the web - Teaser" by openclassrooms on Vimeo, the home for high quality videos and the people who love them.',
1342                 'upload_date': '20151214',
1343                 'uploader': 'OpenClassrooms',
1344                 'uploader_id': 'openclassrooms',
1345             },
1346             'add_ie': ['Vimeo'],
1347         },
1348         {
1349             'url': 'https://support.arkena.com/display/PLAY/Ways+to+embed+your+video',
1350             'md5': 'b96f2f71b359a8ecd05ce4e1daa72365',
1351             'info_dict': {
1352                 'id': 'b41dda37-d8e7-4d3f-b1b5-9a9db578bdfe',
1353                 'ext': 'mp4',
1354                 'title': 'Big Buck Bunny',
1355                 'description': 'Royalty free test video',
1356                 'timestamp': 1432816365,
1357                 'upload_date': '20150528',
1358                 'is_live': False,
1359             },
1360             'params': {
1361                 'skip_download': True,
1362             },
1363             'add_ie': [ArkenaIE.ie_key()],
1364         },
1365         # {
1366         #     # TODO: find another test
1367         #     # http://schema.org/VideoObject
1368         #     'url': 'https://flipagram.com/f/nyvTSJMKId',
1369         #     'md5': '888dcf08b7ea671381f00fab74692755',
1370         #     'info_dict': {
1371         #         'id': 'nyvTSJMKId',
1372         #         'ext': 'mp4',
1373         #         'title': 'Flipagram by sjuria101 featuring Midnight Memories by One Direction',
1374         #         'description': '#love for cats.',
1375         #         'timestamp': 1461244995,
1376         #         'upload_date': '20160421',
1377         #     },
1378         #     'params': {
1379         #         'force_generic_extractor': True,
1380         #     },
1381         # }
1382     ]
1383
1384     def report_following_redirect(self, new_url):
1385         """Report information extraction."""
1386         self._downloader.to_screen('[redirect] Following redirect to %s' % new_url)
1387
1388     def _extract_rss(self, url, video_id, doc):
1389         playlist_title = doc.find('./channel/title').text
1390         playlist_desc_el = doc.find('./channel/description')
1391         playlist_desc = None if playlist_desc_el is None else playlist_desc_el.text
1392
1393         entries = []
1394         for it in doc.findall('./channel/item'):
1395             next_url = xpath_text(it, 'link', fatal=False)
1396             if not next_url:
1397                 enclosure_nodes = it.findall('./enclosure')
1398                 for e in enclosure_nodes:
1399                     next_url = e.attrib.get('url')
1400                     if next_url:
1401                         break
1402
1403             if not next_url:
1404                 continue
1405
1406             entries.append({
1407                 '_type': 'url',
1408                 'url': next_url,
1409                 'title': it.find('title').text,
1410             })
1411
1412         return {
1413             '_type': 'playlist',
1414             'id': url,
1415             'title': playlist_title,
1416             'description': playlist_desc,
1417             'entries': entries,
1418         }
1419
1420     def _extract_camtasia(self, url, video_id, webpage):
1421         """ Returns None if no camtasia video can be found. """
1422
1423         camtasia_cfg = self._search_regex(
1424             r'fo\.addVariable\(\s*"csConfigFile",\s*"([^"]+)"\s*\);',
1425             webpage, 'camtasia configuration file', default=None)
1426         if camtasia_cfg is None:
1427             return None
1428
1429         title = self._html_search_meta('DC.title', webpage, fatal=True)
1430
1431         camtasia_url = compat_urlparse.urljoin(url, camtasia_cfg)
1432         camtasia_cfg = self._download_xml(
1433             camtasia_url, video_id,
1434             note='Downloading camtasia configuration',
1435             errnote='Failed to download camtasia configuration')
1436         fileset_node = camtasia_cfg.find('./playlist/array/fileset')
1437
1438         entries = []
1439         for n in fileset_node.getchildren():
1440             url_n = n.find('./uri')
1441             if url_n is None:
1442                 continue
1443
1444             entries.append({
1445                 'id': os.path.splitext(url_n.text.rpartition('/')[2])[0],
1446                 'title': '%s - %s' % (title, n.tag),
1447                 'url': compat_urlparse.urljoin(url, url_n.text),
1448                 'duration': float_or_none(n.find('./duration').text),
1449             })
1450
1451         return {
1452             '_type': 'playlist',
1453             'entries': entries,
1454             'title': title,
1455         }
1456
1457     def _real_extract(self, url):
1458         if url.startswith('//'):
1459             return {
1460                 '_type': 'url',
1461                 'url': self.http_scheme() + url,
1462             }
1463
1464         parsed_url = compat_urlparse.urlparse(url)
1465         if not parsed_url.scheme:
1466             default_search = self._downloader.params.get('default_search')
1467             if default_search is None:
1468                 default_search = 'fixup_error'
1469
1470             if default_search in ('auto', 'auto_warning', 'fixup_error'):
1471                 if '/' in url:
1472                     self._downloader.report_warning('The url doesn\'t specify the protocol, trying with http')
1473                     return self.url_result('http://' + url)
1474                 elif default_search != 'fixup_error':
1475                     if default_search == 'auto_warning':
1476                         if re.match(r'^(?:url|URL)$', url):
1477                             raise ExtractorError(
1478                                 'Invalid URL:  %r . Call youtube-dl like this:  youtube-dl -v "https://www.youtube.com/watch?v=BaW_jenozKc"  ' % url,
1479                                 expected=True)
1480                         else:
1481                             self._downloader.report_warning(
1482                                 'Falling back to youtube search for  %s . Set --default-search "auto" to suppress this warning.' % url)
1483                     return self.url_result('ytsearch:' + url)
1484
1485             if default_search in ('error', 'fixup_error'):
1486                 raise ExtractorError(
1487                     '%r is not a valid URL. '
1488                     'Set --default-search "ytsearch" (or run  youtube-dl "ytsearch:%s" ) to search YouTube'
1489                     % (url, url), expected=True)
1490             else:
1491                 if ':' not in default_search:
1492                     default_search += ':'
1493                 return self.url_result(default_search + url)
1494
1495         url, smuggled_data = unsmuggle_url(url)
1496         force_videoid = None
1497         is_intentional = smuggled_data and smuggled_data.get('to_generic')
1498         if smuggled_data and 'force_videoid' in smuggled_data:
1499             force_videoid = smuggled_data['force_videoid']
1500             video_id = force_videoid
1501         else:
1502             video_id = compat_urllib_parse_unquote(os.path.splitext(url.rstrip('/').split('/')[-1])[0])
1503
1504         self.to_screen('%s: Requesting header' % video_id)
1505
1506         head_req = HEADRequest(url)
1507         head_response = self._request_webpage(
1508             head_req, video_id,
1509             note=False, errnote='Could not send HEAD request to %s' % url,
1510             fatal=False)
1511
1512         if head_response is not False:
1513             # Check for redirect
1514             new_url = head_response.geturl()
1515             if url != new_url:
1516                 self.report_following_redirect(new_url)
1517                 if force_videoid:
1518                     new_url = smuggle_url(
1519                         new_url, {'force_videoid': force_videoid})
1520                 return self.url_result(new_url)
1521
1522         full_response = None
1523         if head_response is False:
1524             request = sanitized_Request(url)
1525             request.add_header('Accept-Encoding', '*')
1526             full_response = self._request_webpage(request, video_id)
1527             head_response = full_response
1528
1529         info_dict = {
1530             'id': video_id,
1531             'title': compat_urllib_parse_unquote(os.path.splitext(url_basename(url))[0]),
1532             'upload_date': unified_strdate(head_response.headers.get('Last-Modified'))
1533         }
1534
1535         # Check for direct link to a video
1536         content_type = head_response.headers.get('Content-Type', '').lower()
1537         m = re.match(r'^(?P<type>audio|video|application(?=/(?:ogg$|(?:vnd\.apple\.|x-)?mpegurl)))/(?P<format_id>[^;\s]+)', content_type)
1538         if m:
1539             format_id = m.group('format_id')
1540             if format_id.endswith('mpegurl'):
1541                 formats = self._extract_m3u8_formats(url, video_id, 'mp4')
1542             elif format_id == 'f4m':
1543                 formats = self._extract_f4m_formats(url, video_id)
1544             else:
1545                 formats = [{
1546                     'format_id': m.group('format_id'),
1547                     'url': url,
1548                     'vcodec': 'none' if m.group('type') == 'audio' else None
1549                 }]
1550                 info_dict['direct'] = True
1551             self._sort_formats(formats)
1552             info_dict['formats'] = formats
1553             return info_dict
1554
1555         if not self._downloader.params.get('test', False) and not is_intentional:
1556             force = self._downloader.params.get('force_generic_extractor', False)
1557             self._downloader.report_warning(
1558                 '%s on generic information extractor.' % ('Forcing' if force else 'Falling back'))
1559
1560         if not full_response:
1561             request = sanitized_Request(url)
1562             # Some webservers may serve compressed content of rather big size (e.g. gzipped flac)
1563             # making it impossible to download only chunk of the file (yet we need only 512kB to
1564             # test whether it's HTML or not). According to youtube-dl default Accept-Encoding
1565             # that will always result in downloading the whole file that is not desirable.
1566             # Therefore for extraction pass we have to override Accept-Encoding to any in order
1567             # to accept raw bytes and being able to download only a chunk.
1568             # It may probably better to solve this by checking Content-Type for application/octet-stream
1569             # after HEAD request finishes, but not sure if we can rely on this.
1570             request.add_header('Accept-Encoding', '*')
1571             full_response = self._request_webpage(request, video_id)
1572
1573         first_bytes = full_response.read(512)
1574
1575         # Is it an M3U playlist?
1576         if first_bytes.startswith(b'#EXTM3U'):
1577             info_dict['formats'] = self._extract_m3u8_formats(url, video_id, 'mp4')
1578             self._sort_formats(info_dict['formats'])
1579             return info_dict
1580
1581         # Maybe it's a direct link to a video?
1582         # Be careful not to download the whole thing!
1583         if not is_html(first_bytes):
1584             self._downloader.report_warning(
1585                 'URL could be a direct video link, returning it as such.')
1586             info_dict.update({
1587                 'direct': True,
1588                 'url': url,
1589             })
1590             return info_dict
1591
1592         webpage = self._webpage_read_content(
1593             full_response, url, video_id, prefix=first_bytes)
1594
1595         self.report_extraction(video_id)
1596
1597         # Is it an RSS feed, a SMIL file, an XSPF playlist or a MPD manifest?
1598         try:
1599             doc = compat_etree_fromstring(webpage.encode('utf-8'))
1600             if doc.tag == 'rss':
1601                 return self._extract_rss(url, video_id, doc)
1602             elif re.match(r'^(?:{[^}]+})?smil$', doc.tag):
1603                 smil = self._parse_smil(doc, url, video_id)
1604                 self._sort_formats(smil['formats'])
1605                 return smil
1606             elif doc.tag == '{http://xspf.org/ns/0/}playlist':
1607                 return self.playlist_result(self._parse_xspf(doc, video_id), video_id)
1608             elif re.match(r'(?i)^(?:{[^}]+})?MPD$', doc.tag):
1609                 info_dict['formats'] = self._parse_mpd_formats(
1610                     doc, video_id, mpd_base_url=url.rpartition('/')[0])
1611                 self._sort_formats(info_dict['formats'])
1612                 return info_dict
1613             elif re.match(r'^{http://ns\.adobe\.com/f4m/[12]\.0}manifest$', doc.tag):
1614                 info_dict['formats'] = self._parse_f4m_formats(doc, url, video_id)
1615                 self._sort_formats(info_dict['formats'])
1616                 return info_dict
1617         except compat_xml_parse_error:
1618             pass
1619
1620         # Is it a Camtasia project?
1621         camtasia_res = self._extract_camtasia(url, video_id, webpage)
1622         if camtasia_res is not None:
1623             return camtasia_res
1624
1625         # Sometimes embedded video player is hidden behind percent encoding
1626         # (e.g. https://github.com/rg3/youtube-dl/issues/2448)
1627         # Unescaping the whole page allows to handle those cases in a generic way
1628         webpage = compat_urllib_parse_unquote(webpage)
1629
1630         # it's tempting to parse this further, but you would
1631         # have to take into account all the variations like
1632         #   Video Title - Site Name
1633         #   Site Name | Video Title
1634         #   Video Title - Tagline | Site Name
1635         # and so on and so forth; it's just not practical
1636         video_title = self._og_search_title(
1637             webpage, default=None) or self._html_search_regex(
1638             r'(?s)<title>(.*?)</title>', webpage, 'video title',
1639             default='video')
1640
1641         # Try to detect age limit automatically
1642         age_limit = self._rta_search(webpage)
1643         # And then there are the jokers who advertise that they use RTA,
1644         # but actually don't.
1645         AGE_LIMIT_MARKERS = [
1646             r'Proudly Labeled <a href="http://www.rtalabel.org/" title="Restricted to Adults">RTA</a>',
1647         ]
1648         if any(re.search(marker, webpage) for marker in AGE_LIMIT_MARKERS):
1649             age_limit = 18
1650
1651         # video uploader is domain name
1652         video_uploader = self._search_regex(
1653             r'^(?:https?://)?([^/]*)/.*', url, 'video uploader')
1654
1655         video_description = self._og_search_description(webpage, default=None)
1656         video_thumbnail = self._og_search_thumbnail(webpage, default=None)
1657
1658         # Helper method
1659         def _playlist_from_matches(matches, getter=None, ie=None):
1660             urlrs = orderedSet(
1661                 self.url_result(self._proto_relative_url(getter(m) if getter else m), ie)
1662                 for m in matches)
1663             return self.playlist_result(
1664                 urlrs, playlist_id=video_id, playlist_title=video_title)
1665
1666         # Look for Brightcove Legacy Studio embeds
1667         bc_urls = BrightcoveLegacyIE._extract_brightcove_urls(webpage)
1668         if bc_urls:
1669             self.to_screen('Brightcove video detected.')
1670             entries = [{
1671                 '_type': 'url',
1672                 'url': smuggle_url(bc_url, {'Referer': url}),
1673                 'ie_key': 'BrightcoveLegacy'
1674             } for bc_url in bc_urls]
1675
1676             return {
1677                 '_type': 'playlist',
1678                 'title': video_title,
1679                 'id': video_id,
1680                 'entries': entries,
1681             }
1682
1683         # Look for Brightcove New Studio embeds
1684         bc_urls = BrightcoveNewIE._extract_urls(webpage)
1685         if bc_urls:
1686             return _playlist_from_matches(bc_urls, ie='BrightcoveNew')
1687
1688         # Look for ThePlatform embeds
1689         tp_urls = ThePlatformIE._extract_urls(webpage)
1690         if tp_urls:
1691             return _playlist_from_matches(tp_urls, ie='ThePlatform')
1692
1693         # Look for Vessel embeds
1694         vessel_urls = VesselIE._extract_urls(webpage)
1695         if vessel_urls:
1696             return _playlist_from_matches(vessel_urls, ie=VesselIE.ie_key())
1697
1698         # Look for embedded rtl.nl player
1699         matches = re.findall(
1700             r'<iframe[^>]+?src="((?:https?:)?//(?:www\.)?rtl\.nl/system/videoplayer/[^"]+(?:video_)?embed[^"]+)"',
1701             webpage)
1702         if matches:
1703             return _playlist_from_matches(matches, ie='RtlNl')
1704
1705         vimeo_url = VimeoIE._extract_vimeo_url(url, webpage)
1706         if vimeo_url is not None:
1707             return self.url_result(vimeo_url)
1708
1709         vid_me_embed_url = self._search_regex(
1710             r'src=[\'"](https?://vid\.me/[^\'"]+)[\'"]',
1711             webpage, 'vid.me embed', default=None)
1712         if vid_me_embed_url is not None:
1713             return self.url_result(vid_me_embed_url, 'Vidme')
1714
1715         # Look for embedded YouTube player
1716         matches = re.findall(r'''(?x)
1717             (?:
1718                 <iframe[^>]+?src=|
1719                 data-video-url=|
1720                 <embed[^>]+?src=|
1721                 embedSWF\(?:\s*|
1722                 new\s+SWFObject\(
1723             )
1724             (["\'])
1725                 (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
1726                 (?:embed|v|p)/.+?)
1727             \1''', webpage)
1728         if matches:
1729             return _playlist_from_matches(
1730                 matches, lambda m: unescapeHTML(m[1]))
1731
1732         # Look for lazyYT YouTube embed
1733         matches = re.findall(
1734             r'class="lazyYT" data-youtube-id="([^"]+)"', webpage)
1735         if matches:
1736             return _playlist_from_matches(matches, lambda m: unescapeHTML(m))
1737
1738         # Look for Wordpress "YouTube Video Importer" plugin
1739         matches = re.findall(r'''(?x)<div[^>]+
1740             class=(?P<q1>[\'"])[^\'"]*\byvii_single_video_player\b[^\'"]*(?P=q1)[^>]+
1741             data-video_id=(?P<q2>[\'"])([^\'"]+)(?P=q2)''', webpage)
1742         if matches:
1743             return _playlist_from_matches(matches, lambda m: m[-1])
1744
1745         matches = DailymotionIE._extract_urls(webpage)
1746         if matches:
1747             return _playlist_from_matches(matches)
1748
1749         # Look for embedded Dailymotion playlist player (#3822)
1750         m = re.search(
1751             r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?dailymotion\.[a-z]{2,3}/widget/jukebox\?.+?)\1', webpage)
1752         if m:
1753             playlists = re.findall(
1754                 r'list\[\]=/playlist/([^/]+)/', unescapeHTML(m.group('url')))
1755             if playlists:
1756                 return _playlist_from_matches(
1757                     playlists, lambda p: '//dailymotion.com/playlist/%s' % p)
1758
1759         # Look for embedded Wistia player
1760         match = re.search(
1761             r'<(?:meta[^>]+?content|iframe[^>]+?src)=(["\'])(?P<url>(?:https?:)?//(?:fast\.)?wistia\.net/embed/iframe/.+?)\1', webpage)
1762         if match:
1763             embed_url = self._proto_relative_url(
1764                 unescapeHTML(match.group('url')))
1765             return {
1766                 '_type': 'url_transparent',
1767                 'url': embed_url,
1768                 'ie_key': 'Wistia',
1769                 'uploader': video_uploader,
1770             }
1771
1772         match = re.search(r'(?:id=["\']wistia_|data-wistia-?id=["\']|Wistia\.embed\(["\'])(?P<id>[^"\']+)', webpage)
1773         if match:
1774             return {
1775                 '_type': 'url_transparent',
1776                 'url': 'wistia:%s' % match.group('id'),
1777                 'ie_key': 'Wistia',
1778                 'uploader': video_uploader,
1779             }
1780
1781         match = re.search(
1782             r'''(?sx)
1783                 <script[^>]+src=(["'])(?:https?:)?//fast\.wistia\.com/assets/external/E-v1\.js\1[^>]*>.*?
1784                 <div[^>]+class=(["']).*?\bwistia_async_(?P<id>[a-z0-9]+)\b.*?\2
1785             ''', webpage)
1786         if match:
1787             return self.url_result(self._proto_relative_url(
1788                 'wistia:%s' % match.group('id')), 'Wistia')
1789
1790         # Look for SVT player
1791         svt_url = SVTIE._extract_url(webpage)
1792         if svt_url:
1793             return self.url_result(svt_url, 'SVT')
1794
1795         # Look for embedded condenast player
1796         matches = re.findall(
1797             r'<iframe\s+(?:[a-zA-Z-]+="[^"]+"\s+)*?src="(https?://player\.cnevids\.com/embed/[^"]+")',
1798             webpage)
1799         if matches:
1800             return {
1801                 '_type': 'playlist',
1802                 'entries': [{
1803                     '_type': 'url',
1804                     'ie_key': 'CondeNast',
1805                     'url': ma,
1806                 } for ma in matches],
1807                 'title': video_title,
1808                 'id': video_id,
1809             }
1810
1811         # Look for Bandcamp pages with custom domain
1812         mobj = re.search(r'<meta property="og:url"[^>]*?content="(.*?bandcamp\.com.*?)"', webpage)
1813         if mobj is not None:
1814             burl = unescapeHTML(mobj.group(1))
1815             # Don't set the extractor because it can be a track url or an album
1816             return self.url_result(burl)
1817
1818         # Look for embedded Vevo player
1819         mobj = re.search(
1820             r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:cache\.)?vevo\.com/.+?)\1', webpage)
1821         if mobj is not None:
1822             return self.url_result(mobj.group('url'))
1823
1824         # Look for embedded Viddler player
1825         mobj = re.search(
1826             r'<(?:iframe[^>]+?src|param[^>]+?value)=(["\'])(?P<url>(?:https?:)?//(?:www\.)?viddler\.com/(?:embed|player)/.+?)\1',
1827             webpage)
1828         if mobj is not None:
1829             return self.url_result(mobj.group('url'))
1830
1831         # Look for NYTimes player
1832         mobj = re.search(
1833             r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//graphics8\.nytimes\.com/bcvideo/[^/]+/iframe/embed\.html.+?)\1>',
1834             webpage)
1835         if mobj is not None:
1836             return self.url_result(mobj.group('url'))
1837
1838         # Look for Libsyn player
1839         mobj = re.search(
1840             r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//html5-player\.libsyn\.com/embed/.+?)\1', webpage)
1841         if mobj is not None:
1842             return self.url_result(mobj.group('url'))
1843
1844         # Look for Ooyala videos
1845         mobj = (re.search(r'player\.ooyala\.com/[^"?]+[?#][^"]*?(?:embedCode|ec)=(?P<ec>[^"&]+)', webpage) or
1846                 re.search(r'OO\.Player\.create\([\'"].*?[\'"],\s*[\'"](?P<ec>.{32})[\'"]', webpage) or
1847                 re.search(r'SBN\.VideoLinkset\.ooyala\([\'"](?P<ec>.{32})[\'"]\)', webpage) or
1848                 re.search(r'data-ooyala-video-id\s*=\s*[\'"](?P<ec>.{32})[\'"]', webpage))
1849         if mobj is not None:
1850             return OoyalaIE._build_url_result(smuggle_url(mobj.group('ec'), {'domain': url}))
1851
1852         # Look for multiple Ooyala embeds on SBN network websites
1853         mobj = re.search(r'SBN\.VideoLinkset\.entryGroup\((\[.*?\])', webpage)
1854         if mobj is not None:
1855             embeds = self._parse_json(mobj.group(1), video_id, fatal=False)
1856             if embeds:
1857                 return _playlist_from_matches(
1858                     embeds, getter=lambda v: OoyalaIE._url_for_embed_code(smuggle_url(v['provider_video_id'], {'domain': url})), ie='Ooyala')
1859
1860         # Look for Aparat videos
1861         mobj = re.search(r'<iframe .*?src="(http://www\.aparat\.com/video/[^"]+)"', webpage)
1862         if mobj is not None:
1863             return self.url_result(mobj.group(1), 'Aparat')
1864
1865         # Look for MPORA videos
1866         mobj = re.search(r'<iframe .*?src="(http://mpora\.(?:com|de)/videos/[^"]+)"', webpage)
1867         if mobj is not None:
1868             return self.url_result(mobj.group(1), 'Mpora')
1869
1870         # Look for embedded NovaMov-based player
1871         mobj = re.search(
1872             r'''(?x)<(?:pagespeed_)?iframe[^>]+?src=(["\'])
1873                     (?P<url>http://(?:(?:embed|www)\.)?
1874                         (?:novamov\.com|
1875                            nowvideo\.(?:ch|sx|eu|at|ag|co)|
1876                            videoweed\.(?:es|com)|
1877                            movshare\.(?:net|sx|ag)|
1878                            divxstage\.(?:eu|net|ch|co|at|ag))
1879                         /embed\.php.+?)\1''', webpage)
1880         if mobj is not None:
1881             return self.url_result(mobj.group('url'))
1882
1883         # Look for embedded Facebook player
1884         facebook_url = FacebookIE._extract_url(webpage)
1885         if facebook_url is not None:
1886             return self.url_result(facebook_url, 'Facebook')
1887
1888         # Look for embedded VK player
1889         mobj = re.search(r'<iframe[^>]+?src=(["\'])(?P<url>https?://vk\.com/video_ext\.php.+?)\1', webpage)
1890         if mobj is not None:
1891             return self.url_result(mobj.group('url'), 'VK')
1892
1893         # Look for embedded Odnoklassniki player
1894         mobj = re.search(r'<iframe[^>]+?src=(["\'])(?P<url>https?://(?:odnoklassniki|ok)\.ru/videoembed/.+?)\1', webpage)
1895         if mobj is not None:
1896             return self.url_result(mobj.group('url'), 'Odnoklassniki')
1897
1898         # Look for embedded ivi player
1899         mobj = re.search(r'<embed[^>]+?src=(["\'])(?P<url>https?://(?:www\.)?ivi\.ru/video/player.+?)\1', webpage)
1900         if mobj is not None:
1901             return self.url_result(mobj.group('url'), 'Ivi')
1902
1903         # Look for embedded Huffington Post player
1904         mobj = re.search(
1905             r'<iframe[^>]+?src=(["\'])(?P<url>https?://embed\.live\.huffingtonpost\.com/.+?)\1', webpage)
1906         if mobj is not None:
1907             return self.url_result(mobj.group('url'), 'HuffPost')
1908
1909         # Look for embed.ly
1910         mobj = re.search(r'class=["\']embedly-card["\'][^>]href=["\'](?P<url>[^"\']+)', webpage)
1911         if mobj is not None:
1912             return self.url_result(mobj.group('url'))
1913         mobj = re.search(r'class=["\']embedly-embed["\'][^>]src=["\'][^"\']*url=(?P<url>[^&]+)', webpage)
1914         if mobj is not None:
1915             return self.url_result(compat_urllib_parse_unquote(mobj.group('url')))
1916
1917         # Look for funnyordie embed
1918         matches = re.findall(r'<iframe[^>]+?src="(https?://(?:www\.)?funnyordie\.com/embed/[^"]+)"', webpage)
1919         if matches:
1920             return _playlist_from_matches(
1921                 matches, getter=unescapeHTML, ie='FunnyOrDie')
1922
1923         # Look for BBC iPlayer embed
1924         matches = re.findall(r'setPlaylist\("(https?://www\.bbc\.co\.uk/iplayer/[^/]+/[\da-z]{8})"\)', webpage)
1925         if matches:
1926             return _playlist_from_matches(matches, ie='BBCCoUk')
1927
1928         # Look for embedded RUTV player
1929         rutv_url = RUTVIE._extract_url(webpage)
1930         if rutv_url:
1931             return self.url_result(rutv_url, 'RUTV')
1932
1933         # Look for embedded TVC player
1934         tvc_url = TVCIE._extract_url(webpage)
1935         if tvc_url:
1936             return self.url_result(tvc_url, 'TVC')
1937
1938         # Look for embedded SportBox player
1939         sportbox_urls = SportBoxEmbedIE._extract_urls(webpage)
1940         if sportbox_urls:
1941             return _playlist_from_matches(sportbox_urls, ie='SportBoxEmbed')
1942
1943         # Look for embedded PornHub player
1944         pornhub_url = PornHubIE._extract_url(webpage)
1945         if pornhub_url:
1946             return self.url_result(pornhub_url, 'PornHub')
1947
1948         # Look for embedded XHamster player
1949         xhamster_urls = XHamsterEmbedIE._extract_urls(webpage)
1950         if xhamster_urls:
1951             return _playlist_from_matches(xhamster_urls, ie='XHamsterEmbed')
1952
1953         # Look for embedded TNAFlixNetwork player
1954         tnaflix_urls = TNAFlixNetworkEmbedIE._extract_urls(webpage)
1955         if tnaflix_urls:
1956             return _playlist_from_matches(tnaflix_urls, ie=TNAFlixNetworkEmbedIE.ie_key())
1957
1958         # Look for embedded Tvigle player
1959         mobj = re.search(
1960             r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//cloud\.tvigle\.ru/video/.+?)\1', webpage)
1961         if mobj is not None:
1962             return self.url_result(mobj.group('url'), 'Tvigle')
1963
1964         # Look for embedded TED player
1965         mobj = re.search(
1966             r'<iframe[^>]+?src=(["\'])(?P<url>https?://embed(?:-ssl)?\.ted\.com/.+?)\1', webpage)
1967         if mobj is not None:
1968             return self.url_result(mobj.group('url'), 'TED')
1969
1970         # Look for embedded Ustream videos
1971         mobj = re.search(
1972             r'<iframe[^>]+?src=(["\'])(?P<url>http://www\.ustream\.tv/embed/.+?)\1', webpage)
1973         if mobj is not None:
1974             return self.url_result(mobj.group('url'), 'Ustream')
1975
1976         # Look for embedded arte.tv player
1977         mobj = re.search(
1978             r'<(?:script|iframe) [^>]*?src="(?P<url>http://www\.arte\.tv/(?:playerv2/embed|arte_vp/index)[^"]+)"',
1979             webpage)
1980         if mobj is not None:
1981             return self.url_result(mobj.group('url'), 'ArteTVEmbed')
1982
1983         # Look for embedded francetv player
1984         mobj = re.search(
1985             r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?://)?embed\.francetv\.fr/\?ue=.+?)\1',
1986             webpage)
1987         if mobj is not None:
1988             return self.url_result(mobj.group('url'))
1989
1990         # Look for embedded smotri.com player
1991         smotri_url = SmotriIE._extract_url(webpage)
1992         if smotri_url:
1993             return self.url_result(smotri_url, 'Smotri')
1994
1995         # Look for embedded Myvi.ru player
1996         myvi_url = MyviIE._extract_url(webpage)
1997         if myvi_url:
1998             return self.url_result(myvi_url)
1999
2000         # Look for embedded soundcloud player
2001         mobj = re.search(
2002             r'<iframe\s+(?:[a-zA-Z0-9_-]+="[^"]+"\s+)*src="(?P<url>https?://(?:w\.)?soundcloud\.com/player[^"]+)"',
2003             webpage)
2004         if mobj is not None:
2005             url = unescapeHTML(mobj.group('url'))
2006             return self.url_result(url)
2007
2008         # Look for embedded mtvservices player
2009         mtvservices_url = MTVServicesEmbeddedIE._extract_url(webpage)
2010         if mtvservices_url:
2011             return self.url_result(mtvservices_url, ie='MTVServicesEmbedded')
2012
2013         # Look for embedded yahoo player
2014         mobj = re.search(
2015             r'<iframe[^>]+?src=(["\'])(?P<url>https?://(?:screen|movies)\.yahoo\.com/.+?\.html\?format=embed)\1',
2016             webpage)
2017         if mobj is not None:
2018             return self.url_result(mobj.group('url'), 'Yahoo')
2019
2020         # Look for embedded sbs.com.au player
2021         mobj = re.search(
2022             r'''(?x)
2023             (?:
2024                 <meta\s+property="og:video"\s+content=|
2025                 <iframe[^>]+?src=
2026             )
2027             (["\'])(?P<url>https?://(?:www\.)?sbs\.com\.au/ondemand/video/.+?)\1''',
2028             webpage)
2029         if mobj is not None:
2030             return self.url_result(mobj.group('url'), 'SBS')
2031
2032         # Look for embedded Cinchcast player
2033         mobj = re.search(
2034             r'<iframe[^>]+?src=(["\'])(?P<url>https?://player\.cinchcast\.com/.+?)\1',
2035             webpage)
2036         if mobj is not None:
2037             return self.url_result(mobj.group('url'), 'Cinchcast')
2038
2039         mobj = re.search(
2040             r'<iframe[^>]+?src=(["\'])(?P<url>https?://m(?:lb)?\.mlb\.com/shared/video/embed/embed\.html\?.+?)\1',
2041             webpage)
2042         if not mobj:
2043             mobj = re.search(
2044                 r'data-video-link=["\'](?P<url>http://m.mlb.com/video/[^"\']+)',
2045                 webpage)
2046         if mobj is not None:
2047             return self.url_result(mobj.group('url'), 'MLB')
2048
2049         mobj = re.search(
2050             r'<(?:iframe|script)[^>]+?src=(["\'])(?P<url>%s)\1' % CondeNastIE.EMBED_URL,
2051             webpage)
2052         if mobj is not None:
2053             return self.url_result(self._proto_relative_url(mobj.group('url'), scheme='http:'), 'CondeNast')
2054
2055         mobj = re.search(
2056             r'<iframe[^>]+src="(?P<url>https?://(?:new\.)?livestream\.com/[^"]+/player[^"]+)"',
2057             webpage)
2058         if mobj is not None:
2059             return self.url_result(mobj.group('url'), 'Livestream')
2060
2061         # Look for Zapiks embed
2062         mobj = re.search(
2063             r'<iframe[^>]+src="(?P<url>https?://(?:www\.)?zapiks\.fr/index\.php\?.+?)"', webpage)
2064         if mobj is not None:
2065             return self.url_result(mobj.group('url'), 'Zapiks')
2066
2067         # Look for Kaltura embeds
2068         kaltura_url = KalturaIE._extract_url(webpage)
2069         if kaltura_url:
2070             return self.url_result(smuggle_url(kaltura_url, {'source_url': url}), KalturaIE.ie_key())
2071
2072         # Look for Eagle.Platform embeds
2073         eagleplatform_url = EaglePlatformIE._extract_url(webpage)
2074         if eagleplatform_url:
2075             return self.url_result(eagleplatform_url, EaglePlatformIE.ie_key())
2076
2077         # Look for ClipYou (uses Eagle.Platform) embeds
2078         mobj = re.search(
2079             r'<iframe[^>]+src="https?://(?P<host>media\.clipyou\.ru)/index/player\?.*\brecord_id=(?P<id>\d+).*"', webpage)
2080         if mobj is not None:
2081             return self.url_result('eagleplatform:%(host)s:%(id)s' % mobj.groupdict(), 'EaglePlatform')
2082
2083         # Look for Pladform embeds
2084         pladform_url = PladformIE._extract_url(webpage)
2085         if pladform_url:
2086             return self.url_result(pladform_url)
2087
2088         # Look for Videomore embeds
2089         videomore_url = VideomoreIE._extract_url(webpage)
2090         if videomore_url:
2091             return self.url_result(videomore_url)
2092
2093         # Look for Playwire embeds
2094         mobj = re.search(
2095             r'<script[^>]+data-config=(["\'])(?P<url>(?:https?:)?//config\.playwire\.com/.+?)\1', webpage)
2096         if mobj is not None:
2097             return self.url_result(mobj.group('url'))
2098
2099         # Look for 5min embeds
2100         mobj = re.search(
2101             r'<meta[^>]+property="og:video"[^>]+content="https?://embed\.5min\.com/(?P<id>[0-9]+)/?', webpage)
2102         if mobj is not None:
2103             return self.url_result('5min:%s' % mobj.group('id'), 'FiveMin')
2104
2105         # Look for Crooks and Liars embeds
2106         mobj = re.search(
2107             r'<(?:iframe[^>]+src|param[^>]+value)=(["\'])(?P<url>(?:https?:)?//embed\.crooksandliars\.com/(?:embed|v)/.+?)\1', webpage)
2108         if mobj is not None:
2109             return self.url_result(mobj.group('url'))
2110
2111         # Look for NBC Sports VPlayer embeds
2112         nbc_sports_url = NBCSportsVPlayerIE._extract_url(webpage)
2113         if nbc_sports_url:
2114             return self.url_result(nbc_sports_url, 'NBCSportsVPlayer')
2115
2116         # Look for NBC News embeds
2117         nbc_news_embed_url = re.search(
2118             r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//www\.nbcnews\.com/widget/video-embed/[^"\']+)\1', webpage)
2119         if nbc_news_embed_url:
2120             return self.url_result(nbc_news_embed_url.group('url'), 'NBCNews')
2121
2122         # Look for Google Drive embeds
2123         google_drive_url = GoogleDriveIE._extract_url(webpage)
2124         if google_drive_url:
2125             return self.url_result(google_drive_url, 'GoogleDrive')
2126
2127         # Look for UDN embeds
2128         mobj = re.search(
2129             r'<iframe[^>]+src="(?P<url>%s)"' % UDNEmbedIE._PROTOCOL_RELATIVE_VALID_URL, webpage)
2130         if mobj is not None:
2131             return self.url_result(
2132                 compat_urlparse.urljoin(url, mobj.group('url')), 'UDNEmbed')
2133
2134         # Look for Senate ISVP iframe
2135         senate_isvp_url = SenateISVPIE._search_iframe_url(webpage)
2136         if senate_isvp_url:
2137             return self.url_result(senate_isvp_url, 'SenateISVP')
2138
2139         # Look for Dailymotion Cloud videos
2140         dmcloud_url = DailymotionCloudIE._extract_dmcloud_url(webpage)
2141         if dmcloud_url:
2142             return self.url_result(dmcloud_url, 'DailymotionCloud')
2143
2144         # Look for OnionStudios embeds
2145         onionstudios_url = OnionStudiosIE._extract_url(webpage)
2146         if onionstudios_url:
2147             return self.url_result(onionstudios_url)
2148
2149         # Look for ViewLift embeds
2150         viewlift_url = ViewLiftEmbedIE._extract_url(webpage)
2151         if viewlift_url:
2152             return self.url_result(viewlift_url)
2153
2154         # Look for JWPlatform embeds
2155         jwplatform_url = JWPlatformIE._extract_url(webpage)
2156         if jwplatform_url:
2157             return self.url_result(jwplatform_url, 'JWPlatform')
2158
2159         # Look for ScreenwaveMedia embeds
2160         mobj = re.search(ScreenwaveMediaIE.EMBED_PATTERN, webpage)
2161         if mobj is not None:
2162             return self.url_result(unescapeHTML(mobj.group('url')), 'ScreenwaveMedia')
2163
2164         # Look for Digiteka embeds
2165         digiteka_url = DigitekaIE._extract_url(webpage)
2166         if digiteka_url:
2167             return self.url_result(self._proto_relative_url(digiteka_url), DigitekaIE.ie_key())
2168
2169         # Look for Arkena embeds
2170         arkena_url = ArkenaIE._extract_url(webpage)
2171         if arkena_url:
2172             return self.url_result(arkena_url, ArkenaIE.ie_key())
2173
2174         # Look for Limelight embeds
2175         mobj = re.search(r'LimelightPlayer\.doLoad(Media|Channel|ChannelList)\(["\'](?P<id>[a-z0-9]{32})', webpage)
2176         if mobj:
2177             lm = {
2178                 'Media': 'media',
2179                 'Channel': 'channel',
2180                 'ChannelList': 'channel_list',
2181             }
2182             return self.url_result('limelight:%s:%s' % (
2183                 lm[mobj.group(1)], mobj.group(2)), 'Limelight%s' % mobj.group(1), mobj.group(2))
2184
2185         # Look for AdobeTVVideo embeds
2186         mobj = re.search(
2187             r'<iframe[^>]+src=[\'"]((?:https?:)?//video\.tv\.adobe\.com/v/\d+[^"]+)[\'"]',
2188             webpage)
2189         if mobj is not None:
2190             return self.url_result(
2191                 self._proto_relative_url(unescapeHTML(mobj.group(1))),
2192                 'AdobeTVVideo')
2193
2194         # Look for Vine embeds
2195         mobj = re.search(
2196             r'<iframe[^>]+src=[\'"]((?:https?:)?//(?:www\.)?vine\.co/v/[^/]+/embed/(?:simple|postcard))',
2197             webpage)
2198         if mobj is not None:
2199             return self.url_result(
2200                 self._proto_relative_url(unescapeHTML(mobj.group(1))), 'Vine')
2201
2202         # Look for Instagram embeds
2203         instagram_embed_url = InstagramIE._extract_embed_url(webpage)
2204         if instagram_embed_url is not None:
2205             return self.url_result(
2206                 self._proto_relative_url(instagram_embed_url), InstagramIE.ie_key())
2207
2208         # Look for LiveLeak embeds
2209         liveleak_url = LiveLeakIE._extract_url(webpage)
2210         if liveleak_url:
2211             return self.url_result(liveleak_url, 'LiveLeak')
2212
2213         # Look for 3Q SDN embeds
2214         threeqsdn_url = ThreeQSDNIE._extract_url(webpage)
2215         if threeqsdn_url:
2216             return {
2217                 '_type': 'url_transparent',
2218                 'ie_key': ThreeQSDNIE.ie_key(),
2219                 'url': self._proto_relative_url(threeqsdn_url),
2220                 'title': video_title,
2221                 'description': video_description,
2222                 'thumbnail': video_thumbnail,
2223                 'uploader': video_uploader,
2224             }
2225
2226         # Looking for http://schema.org/VideoObject
2227         json_ld = self._search_json_ld(
2228             webpage, video_id, default=None, expected_type='VideoObject')
2229         if json_ld and json_ld.get('url'):
2230             info_dict.update({
2231                 'title': video_title or info_dict['title'],
2232                 'description': video_description,
2233                 'thumbnail': video_thumbnail,
2234                 'age_limit': age_limit
2235             })
2236             info_dict.update(json_ld)
2237             return info_dict
2238
2239         def check_video(vurl):
2240             if YoutubeIE.suitable(vurl):
2241                 return True
2242             vpath = compat_urlparse.urlparse(vurl).path
2243             vext = determine_ext(vpath)
2244             return '.' in vpath and vext not in ('swf', 'png', 'jpg', 'srt', 'sbv', 'sub', 'vtt', 'ttml')
2245
2246         def filter_video(urls):
2247             return list(filter(check_video, urls))
2248
2249         # Start with something easy: JW Player in SWFObject
2250         found = filter_video(re.findall(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage))
2251         if not found:
2252             # Look for gorilla-vid style embedding
2253             found = filter_video(re.findall(r'''(?sx)
2254                 (?:
2255                     jw_plugins|
2256                     JWPlayerOptions|
2257                     jwplayer\s*\(\s*["'][^'"]+["']\s*\)\s*\.setup
2258                 )
2259                 .*?
2260                 ['"]?file['"]?\s*:\s*["\'](.*?)["\']''', webpage))
2261         if not found:
2262             # Broaden the search a little bit
2263             found = filter_video(re.findall(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage))
2264         if not found:
2265             # Broaden the findall a little bit: JWPlayer JS loader
2266             found = filter_video(re.findall(
2267                 r'[^A-Za-z0-9]?(?:file|video_url)["\']?:\s*["\'](http(?![^\'"]+\.[0-9]+[\'"])[^\'"]+)["\']', webpage))
2268         if not found:
2269             # Flow player
2270             found = filter_video(re.findall(r'''(?xs)
2271                 flowplayer\("[^"]+",\s*
2272                     \{[^}]+?\}\s*,
2273                     \s*\{[^}]+? ["']?clip["']?\s*:\s*\{\s*
2274                         ["']?url["']?\s*:\s*["']([^"']+)["']
2275             ''', webpage))
2276         if not found:
2277             # Cinerama player
2278             found = re.findall(
2279                 r"cinerama\.embedPlayer\(\s*\'[^']+\',\s*'([^']+)'", webpage)
2280         if not found:
2281             # Try to find twitter cards info
2282             # twitter:player:stream should be checked before twitter:player since
2283             # it is expected to contain a raw stream (see
2284             # https://dev.twitter.com/cards/types/player#On_twitter.com_via_desktop_browser)
2285             found = filter_video(re.findall(
2286                 r'<meta (?:property|name)="twitter:player:stream" (?:content|value)="(.+?)"', webpage))
2287         if not found:
2288             # We look for Open Graph info:
2289             # We have to match any number spaces between elements, some sites try to align them (eg.: statigr.am)
2290             m_video_type = re.findall(r'<meta.*?property="og:video:type".*?content="video/(.*?)"', webpage)
2291             # We only look in og:video if the MIME type is a video, don't try if it's a Flash player:
2292             if m_video_type is not None:
2293                 found = filter_video(re.findall(r'<meta.*?property="og:video".*?content="(.*?)"', webpage))
2294         if not found:
2295             # HTML5 video
2296             found = re.findall(r'(?s)<(?:video|audio)[^<]*(?:>.*?<source[^>]*)?\s+src=["\'](.*?)["\']', webpage)
2297         if not found:
2298             REDIRECT_REGEX = r'[0-9]{,2};\s*(?:URL|url)=\'?([^\'"]+)'
2299             found = re.search(
2300                 r'(?i)<meta\s+(?=(?:[a-z-]+="[^"]+"\s+)*http-equiv="refresh")'
2301                 r'(?:[a-z-]+="[^"]+"\s+)*?content="%s' % REDIRECT_REGEX,
2302                 webpage)
2303             if not found:
2304                 # Look also in Refresh HTTP header
2305                 refresh_header = head_response.headers.get('Refresh')
2306                 if refresh_header:
2307                     # In python 2 response HTTP headers are bytestrings
2308                     if sys.version_info < (3, 0) and isinstance(refresh_header, str):
2309                         refresh_header = refresh_header.decode('iso-8859-1')
2310                     found = re.search(REDIRECT_REGEX, refresh_header)
2311             if found:
2312                 new_url = compat_urlparse.urljoin(url, unescapeHTML(found.group(1)))
2313                 self.report_following_redirect(new_url)
2314                 return {
2315                     '_type': 'url',
2316                     'url': new_url,
2317                 }
2318
2319         if not found:
2320             # twitter:player is a https URL to iframe player that may or may not
2321             # be supported by youtube-dl thus this is checked the very last (see
2322             # https://dev.twitter.com/cards/types/player#On_twitter.com_via_desktop_browser)
2323             embed_url = self._html_search_meta('twitter:player', webpage, default=None)
2324             if embed_url:
2325                 return self.url_result(embed_url)
2326
2327         if not found:
2328             raise UnsupportedError(url)
2329
2330         entries = []
2331         for video_url in orderedSet(found):
2332             video_url = unescapeHTML(video_url)
2333             video_url = video_url.replace('\\/', '/')
2334             video_url = compat_urlparse.urljoin(url, video_url)
2335             video_id = compat_urllib_parse_unquote(os.path.basename(video_url))
2336
2337             # Sometimes, jwplayer extraction will result in a YouTube URL
2338             if YoutubeIE.suitable(video_url):
2339                 entries.append(self.url_result(video_url, 'Youtube'))
2340                 continue
2341
2342             # here's a fun little line of code for you:
2343             video_id = os.path.splitext(video_id)[0]
2344
2345             entry_info_dict = {
2346                 'id': video_id,
2347                 'uploader': video_uploader,
2348                 'title': video_title,
2349                 'age_limit': age_limit,
2350             }
2351
2352             ext = determine_ext(video_url)
2353             if ext == 'smil':
2354                 entry_info_dict['formats'] = self._extract_smil_formats(video_url, video_id)
2355             elif ext == 'xspf':
2356                 return self.playlist_result(self._extract_xspf_playlist(video_url, video_id), video_id)
2357             elif ext == 'm3u8':
2358                 entry_info_dict['formats'] = self._extract_m3u8_formats(video_url, video_id, ext='mp4')
2359             elif ext == 'mpd':
2360                 entry_info_dict['formats'] = self._extract_mpd_formats(video_url, video_id)
2361             elif ext == 'f4m':
2362                 entry_info_dict['formats'] = self._extract_f4m_formats(video_url, video_id)
2363             else:
2364                 entry_info_dict['url'] = video_url
2365
2366             if entry_info_dict.get('formats'):
2367                 self._sort_formats(entry_info_dict['formats'])
2368
2369             entries.append(entry_info_dict)
2370
2371         if len(entries) == 1:
2372             return entries[0]
2373         else:
2374             for num, e in enumerate(entries, start=1):
2375                 # 'url' results don't have a title
2376                 if e.get('title') is not None:
2377                     e['title'] = '%s (%d)' % (e['title'], num)
2378             return {
2379                 '_type': 'playlist',
2380                 'entries': entries,
2381             }