youtube_dl/extractor/generic.py

   1 # coding: utf-8
   2
   3 from __future__ import unicode_literals
   4
   5 import os
   6 import re
   7 import sys
   8
   9 from .common import InfoExtractor
  10 from .youtube import YoutubeIE
  11 from ..compat import (
  12     compat_etree_fromstring,
  13     compat_urllib_parse_unquote,
  14     compat_urlparse,
  15     compat_xml_parse_error,
  16 )
  17 from ..utils import (
  18     determine_ext,
  19     ExtractorError,
  20     float_or_none,
  21     HEADRequest,
  22     is_html,
  23     orderedSet,
  24     sanitized_Request,
  25     smuggle_url,
  26     unescapeHTML,
  27     unified_strdate,
  28     unsmuggle_url,
  29     UnsupportedError,
  30     xpath_text,
  31 )
  32 from .brightcove import (
  33     BrightcoveLegacyIE,
  34     BrightcoveNewIE,
  35 )
  36 from .nbc import NBCSportsVPlayerIE
  37 from .ooyala import OoyalaIE
  38 from .rutv import RUTVIE
  39 from .tvc import TVCIE
  40 from .sportbox import SportBoxEmbedIE
  41 from .smotri import SmotriIE
  42 from .myvi import MyviIE
  43 from .condenast import CondeNastIE
  44 from .udn import UDNEmbedIE
  45 from .senateisvp import SenateISVPIE
  46 from .svt import SVTIE
  47 from .pornhub import PornHubIE
  48 from .xhamster import XHamsterEmbedIE
  49 from .tnaflix import TNAFlixNetworkEmbedIE
  50 from .vimeo import VimeoIE
  51 from .dailymotion import (
  52     DailymotionIE,
  53     DailymotionCloudIE,
  54 )
  55 from .onionstudios import OnionStudiosIE
  56 from .viewlift import ViewLiftEmbedIE
  57 from .screenwavemedia import ScreenwaveMediaIE
  58 from .mtv import MTVServicesEmbeddedIE
  59 from .pladform import PladformIE
  60 from .videomore import VideomoreIE
  61 from .googledrive import GoogleDriveIE
  62 from .jwplatform import JWPlatformIE
  63 from .digiteka import DigitekaIE
  64 from .arkena import ArkenaIE
  65 from .instagram import InstagramIE
  66 from .liveleak import LiveLeakIE
  67 from .threeqsdn import ThreeQSDNIE
  68 from .theplatform import ThePlatformIE
  69 from .vessel import VesselIE
  70 from .kaltura import KalturaIE
  71 from .eagleplatform import EaglePlatformIE
  72 from .facebook import FacebookIE
  73 from .soundcloud import SoundcloudIE
  74 from .vbox7 import Vbox7IE
  75 from .dbtv import DBTVIE
  76
  77
  78 class GenericIE(InfoExtractor):
  79     IE_DESC = 'Generic downloader that works on some sites'
  80     _VALID_URL = r'.*'
  81     IE_NAME = 'generic'
  82     _TESTS = [
  83         # Direct link to a video
  84         {
  85             'url': 'http://media.w3.org/2010/05/sintel/trailer.mp4',
  86             'md5': '67d406c2bcb6af27fa886f31aa934bbe',
  87             'info_dict': {
  88                 'id': 'trailer',
  89                 'ext': 'mp4',
  90                 'title': 'trailer',
  91                 'upload_date': '20100513',
  92             }
  93         },
  94         # Direct link to media delivered compressed (until Accept-Encoding is *)
  95         {
  96             'url': 'http://calimero.tk/muzik/FictionJunction-Parallel_Hearts.flac',
  97             'md5': '128c42e68b13950268b648275386fc74',
  98             'info_dict': {
  99                 'id': 'FictionJunction-Parallel_Hearts',
 100                 'ext': 'flac',
 101                 'title': 'FictionJunction-Parallel_Hearts',
 102                 'upload_date': '20140522',
 103             },
 104             'expected_warnings': [
 105                 'URL could be a direct video link, returning it as such.'
 106             ],
 107             'skip': 'URL invalid',
 108         },
 109         # Direct download with broken HEAD
 110         {
 111             'url': 'http://ai-radio.org:8000/radio.opus',
 112             'info_dict': {
 113                 'id': 'radio',
 114                 'ext': 'opus',
 115                 'title': 'radio',
 116             },
 117             'params': {
 118                 'skip_download': True,  # infinite live stream
 119             },
 120             'expected_warnings': [
 121                 r'501.*Not Implemented',
 122                 r'400.*Bad Request',
 123             ],
 124         },
 125         # Direct link with incorrect MIME type
 126         {
 127             'url': 'http://ftp.nluug.nl/video/nluug/2014-11-20_nj14/zaal-2/5_Lennart_Poettering_-_Systemd.webm',
 128             'md5': '4ccbebe5f36706d85221f204d7eb5913',
 129             'info_dict': {
 130                 'url': 'http://ftp.nluug.nl/video/nluug/2014-11-20_nj14/zaal-2/5_Lennart_Poettering_-_Systemd.webm',
 131                 'id': '5_Lennart_Poettering_-_Systemd',
 132                 'ext': 'webm',
 133                 'title': '5_Lennart_Poettering_-_Systemd',
 134                 'upload_date': '20141120',
 135             },
 136             'expected_warnings': [
 137                 'URL could be a direct video link, returning it as such.'
 138             ]
 139         },
 140         # RSS feed
 141         {
 142             'url': 'http://phihag.de/2014/youtube-dl/rss2.xml',
 143             'info_dict': {
 144                 'id': 'http://phihag.de/2014/youtube-dl/rss2.xml',
 145                 'title': 'Zero Punctuation',
 146                 'description': 're:.*groundbreaking video review series.*'
 147             },
 148             'playlist_mincount': 11,
 149         },
 150         # RSS feed with enclosure
 151         {
 152             'url': 'http://podcastfeeds.nbcnews.com/audio/podcast/MSNBC-MADDOW-NETCAST-M4V.xml',
 153             'info_dict': {
 154                 'id': 'pdv_maddow_netcast_m4v-02-27-2015-201624',
 155                 'ext': 'm4v',
 156                 'upload_date': '20150228',
 157                 'title': 'pdv_maddow_netcast_m4v-02-27-2015-201624',
 158             }
 159         },
 160         # SMIL from http://videolectures.net/promogram_igor_mekjavic_eng
 161         {
 162             'url': 'http://videolectures.net/promogram_igor_mekjavic_eng/video/1/smil.xml',
 163             'info_dict': {
 164                 'id': 'smil',
 165                 'ext': 'mp4',
 166                 'title': 'Automatics, robotics and biocybernetics',
 167                 'description': 'md5:815fc1deb6b3a2bff99de2d5325be482',
 168                 'upload_date': '20130627',
 169                 'formats': 'mincount:16',
 170                 'subtitles': 'mincount:1',
 171             },
 172             'params': {
 173                 'force_generic_extractor': True,
 174                 'skip_download': True,
 175             },
 176         },
 177         # SMIL from http://www1.wdr.de/mediathek/video/livestream/index.html
 178         {
 179             'url': 'http://metafilegenerator.de/WDR/WDR_FS/hds/hds.smil',
 180             'info_dict': {
 181                 'id': 'hds',
 182                 'ext': 'flv',
 183                 'title': 'hds',
 184                 'formats': 'mincount:1',
 185             },
 186             'params': {
 187                 'skip_download': True,
 188             },
 189         },
 190         # SMIL from https://www.restudy.dk/video/play/id/1637
 191         {
 192             'url': 'https://www.restudy.dk/awsmedia/SmilDirectory/video_1637.xml',
 193             'info_dict': {
 194                 'id': 'video_1637',
 195                 'ext': 'flv',
 196                 'title': 'video_1637',
 197                 'formats': 'mincount:3',
 198             },
 199             'params': {
 200                 'skip_download': True,
 201             },
 202         },
 203         # SMIL from http://adventure.howstuffworks.com/5266-cool-jobs-iditarod-musher-video.htm
 204         {
 205             'url': 'http://services.media.howstuffworks.com/videos/450221/smil-service.smil',
 206             'info_dict': {
 207                 'id': 'smil-service',
 208                 'ext': 'flv',
 209                 'title': 'smil-service',
 210                 'formats': 'mincount:1',
 211             },
 212             'params': {
 213                 'skip_download': True,
 214             },
 215         },
 216         # SMIL from http://new.livestream.com/CoheedandCambria/WebsterHall/videos/4719370
 217         {
 218             'url': 'http://api.new.livestream.com/accounts/1570303/events/1585861/videos/4719370.smil',
 219             'info_dict': {
 220                 'id': '4719370',
 221                 'ext': 'mp4',
 222                 'title': '571de1fd-47bc-48db-abf9-238872a58d1f',
 223                 'formats': 'mincount:3',
 224             },
 225             'params': {
 226                 'skip_download': True,
 227             },
 228         },
 229         # XSPF playlist from http://www.telegraaf.nl/tv/nieuws/binnenland/24353229/__Tikibad_ontruimd_wegens_brand__.html
 230         {
 231             'url': 'http://www.telegraaf.nl/xml/playlist/2015/8/7/mZlp2ctYIUEB.xspf',
 232             'info_dict': {
 233                 'id': 'mZlp2ctYIUEB',
 234                 'ext': 'mp4',
 235                 'title': 'Tikibad ontruimd wegens brand',
 236                 'description': 'md5:05ca046ff47b931f9b04855015e163a4',
 237                 'thumbnail': 're:^https?://.*\.jpg$',
 238                 'duration': 33,
 239             },
 240             'params': {
 241                 'skip_download': True,
 242             },
 243         },
 244         # MPD from http://dash-mse-test.appspot.com/media.html
 245         {
 246             'url': 'http://yt-dash-mse-test.commondatastorage.googleapis.com/media/car-20120827-manifest.mpd',
 247             'md5': '4b57baab2e30d6eb3a6a09f0ba57ef53',
 248             'info_dict': {
 249                 'id': 'car-20120827-manifest',
 250                 'ext': 'mp4',
 251                 'title': 'car-20120827-manifest',
 252                 'formats': 'mincount:9',
 253                 'upload_date': '20130904',
 254             },
 255             'params': {
 256                 'format': 'bestvideo',
 257             },
 258         },
 259         # m3u8 served with Content-Type: audio/x-mpegURL; charset=utf-8
 260         {
 261             'url': 'http://once.unicornmedia.com/now/master/playlist/bb0b18ba-64f5-4b1b-a29f-0ac252f06b68/77a785f3-5188-4806-b788-0893a61634ed/93677179-2d99-4ef4-9e17-fe70d49abfbf/content.m3u8',
 262             'info_dict': {
 263                 'id': 'content',
 264                 'ext': 'mp4',
 265                 'title': 'content',
 266                 'formats': 'mincount:8',
 267             },
 268             'params': {
 269                 # m3u8 downloads
 270                 'skip_download': True,
 271             },
 272             'skip': 'video gone',
 273         },
 274         # m3u8 served with Content-Type: text/plain
 275         {
 276             'url': 'http://www.nacentapps.com/m3u8/index.m3u8',
 277             'info_dict': {
 278                 'id': 'index',
 279                 'ext': 'mp4',
 280                 'title': 'index',
 281                 'upload_date': '20140720',
 282                 'formats': 'mincount:11',
 283             },
 284             'params': {
 285                 # m3u8 downloads
 286                 'skip_download': True,
 287             },
 288             'skip': 'video gone',
 289         },
 290         # google redirect
 291         {
 292             'url': 'http://www.google.com/url?sa=t&rct=j&q=&esrc=s&source=web&cd=1&cad=rja&ved=0CCUQtwIwAA&url=http%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DcmQHVoWB5FY&ei=F-sNU-LLCaXk4QT52ICQBQ&usg=AFQjCNEw4hL29zgOohLXvpJ-Bdh2bils1Q&bvm=bv.61965928,d.bGE',
 293             'info_dict': {
 294                 'id': 'cmQHVoWB5FY',
 295                 'ext': 'mp4',
 296                 'upload_date': '20130224',
 297                 'uploader_id': 'TheVerge',
 298                 'description': 're:^Chris Ziegler takes a look at the\.*',
 299                 'uploader': 'The Verge',
 300                 'title': 'First Firefox OS phones side-by-side',
 301             },
 302             'params': {
 303                 'skip_download': False,
 304             }
 305         },
 306         {
 307             # redirect in Refresh HTTP header
 308             'url': 'https://www.facebook.com/l.php?u=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DpO8h3EaFRdo&h=TAQHsoToz&enc=AZN16h-b6o4Zq9pZkCCdOLNKMN96BbGMNtcFwHSaazus4JHT_MFYkAA-WARTX2kvsCIdlAIyHZjl6d33ILIJU7Jzwk_K3mcenAXoAzBNoZDI_Q7EXGDJnIhrGkLXo_LJ_pAa2Jzbx17UHMd3jAs--6j2zaeto5w9RTn8T_1kKg3fdC5WPX9Dbb18vzH7YFX0eSJmoa6SP114rvlkw6pkS1-T&s=1',
 309             'info_dict': {
 310                 'id': 'pO8h3EaFRdo',
 311                 'ext': 'mp4',
 312                 'title': 'Tripeo Boiler Room x Dekmantel Festival DJ Set',
 313                 'description': 'md5:6294cc1af09c4049e0652b51a2df10d5',
 314                 'upload_date': '20150917',
 315                 'uploader_id': 'brtvofficial',
 316                 'uploader': 'Boiler Room',
 317             },
 318             'params': {
 319                 'skip_download': False,
 320             },
 321         },
 322         {
 323             'url': 'http://www.hodiho.fr/2013/02/regis-plante-sa-jeep.html',
 324             'md5': '85b90ccc9d73b4acd9138d3af4c27f89',
 325             'info_dict': {
 326                 'id': '13601338388002',
 327                 'ext': 'mp4',
 328                 'uploader': 'www.hodiho.fr',
 329                 'title': 'R\u00e9gis plante sa Jeep',
 330             }
 331         },
 332         # bandcamp page with custom domain
 333         {
 334             'add_ie': ['Bandcamp'],
 335             'url': 'http://bronyrock.com/track/the-pony-mash',
 336             'info_dict': {
 337                 'id': '3235767654',
 338                 'ext': 'mp3',
 339                 'title': 'The Pony Mash',
 340                 'uploader': 'M_Pallante',
 341             },
 342             'skip': 'There is a limit of 200 free downloads / month for the test song',
 343         },
 344         # embedded brightcove video
 345         # it also tests brightcove videos that need to set the 'Referer' in the
 346         # http requests
 347         {
 348             'add_ie': ['BrightcoveLegacy'],
 349             'url': 'http://www.bfmtv.com/video/bfmbusiness/cours-bourse/cours-bourse-l-analyse-technique-154522/',
 350             'info_dict': {
 351                 'id': '2765128793001',
 352                 'ext': 'mp4',
 353                 'title': 'Le cours de bourse : l’analyse technique',
 354                 'description': 'md5:7e9ad046e968cb2d1114004aba466fd9',
 355                 'uploader': 'BFM BUSINESS',
 356             },
 357             'params': {
 358                 'skip_download': True,
 359             },
 360         },
 361         {
 362             # https://github.com/rg3/youtube-dl/issues/2253
 363             'url': 'http://bcove.me/i6nfkrc3',
 364             'md5': '0ba9446db037002366bab3b3eb30c88c',
 365             'info_dict': {
 366                 'id': '3101154703001',
 367                 'ext': 'mp4',
 368                 'title': 'Still no power',
 369                 'uploader': 'thestar.com',
 370                 'description': 'Mississauga resident David Farmer is still out of power as a result of the ice storm a month ago. To keep the house warm, Farmer cuts wood from his property for a wood burning stove downstairs.',
 371             },
 372             'add_ie': ['BrightcoveLegacy'],
 373             'skip': 'video gone',
 374         },
 375         {
 376             'url': 'http://www.championat.com/video/football/v/87/87499.html',
 377             'md5': 'fb973ecf6e4a78a67453647444222983',
 378             'info_dict': {
 379                 'id': '3414141473001',
 380                 'ext': 'mp4',
 381                 'title': 'Видео. Удаление Дзагоева (ЦСКА)',
 382                 'description': 'Онлайн-трансляция матча ЦСКА - "Волга"',
 383                 'uploader': 'Championat',
 384             },
 385         },
 386         {
 387             # https://github.com/rg3/youtube-dl/issues/3541
 388             'add_ie': ['BrightcoveLegacy'],
 389             'url': 'http://www.kijk.nl/sbs6/leermijvrouwenkennen/videos/jqMiXKAYan2S/aflevering-1',
 390             'info_dict': {
 391                 'id': '3866516442001',
 392                 'ext': 'mp4',
 393                 'title': 'Leer mij vrouwen kennen: Aflevering 1',
 394                 'description': 'Leer mij vrouwen kennen: Aflevering 1',
 395                 'uploader': 'SBS Broadcasting',
 396             },
 397             'skip': 'Restricted to Netherlands',
 398             'params': {
 399                 'skip_download': True,  # m3u8 download
 400             },
 401         },
 402         # ooyala video
 403         {
 404             'url': 'http://www.rollingstone.com/music/videos/norwegian-dj-cashmere-cat-goes-spartan-on-with-me-premiere-20131219',
 405             'md5': '166dd577b433b4d4ebfee10b0824d8ff',
 406             'info_dict': {
 407                 'id': 'BwY2RxaTrTkslxOfcan0UCf0YqyvWysJ',
 408                 'ext': 'mp4',
 409                 'title': '2cc213299525360.mov',  # that's what we get
 410                 'duration': 238.231,
 411             },
 412             'add_ie': ['Ooyala'],
 413         },
 414         {
 415             # ooyala video embedded with http://player.ooyala.com/iframe.js
 416             'url': 'http://www.macrumors.com/2015/07/24/steve-jobs-the-man-in-the-machine-first-trailer/',
 417             'info_dict': {
 418                 'id': 'p0MGJndjoG5SOKqO_hZJuZFPB-Tr5VgB',
 419                 'ext': 'mp4',
 420                 'title': '"Steve Jobs: Man in the Machine" trailer',
 421                 'description': 'The first trailer for the Alex Gibney documentary "Steve Jobs: Man in the Machine."',
 422                 'duration': 135.427,
 423             },
 424             'params': {
 425                 'skip_download': True,
 426             },
 427             'skip': 'movie expired',
 428         },
 429         # embed.ly video
 430         {
 431             'url': 'http://www.tested.com/science/weird/460206-tested-grinding-coffee-2000-frames-second/',
 432             'info_dict': {
 433                 'id': '9ODmcdjQcHQ',
 434                 'ext': 'mp4',
 435                 'title': 'Tested: Grinding Coffee at 2000 Frames Per Second',
 436                 'upload_date': '20140225',
 437                 'description': 'md5:06a40fbf30b220468f1e0957c0f558ff',
 438                 'uploader': 'Tested',
 439                 'uploader_id': 'testedcom',
 440             },
 441             # No need to test YoutubeIE here
 442             'params': {
 443                 'skip_download': True,
 444             },
 445         },
 446         # funnyordie embed
 447         {
 448             'url': 'http://www.theguardian.com/world/2014/mar/11/obama-zach-galifianakis-between-two-ferns',
 449             'info_dict': {
 450                 'id': '18e820ec3f',
 451                 'ext': 'mp4',
 452                 'title': 'Between Two Ferns with Zach Galifianakis: President Barack Obama',
 453                 'description': 'Episode 18: President Barack Obama sits down with Zach Galifianakis for his most memorable interview yet.',
 454             },
 455             # HEAD requests lead to endless 301, while GET is OK
 456             'expected_warnings': ['301'],
 457         },
 458         # RUTV embed
 459         {
 460             'url': 'http://www.rg.ru/2014/03/15/reg-dfo/anklav-anons.html',
 461             'info_dict': {
 462                 'id': '776940',
 463                 'ext': 'mp4',
 464                 'title': 'Охотское море стало целиком российским',
 465                 'description': 'md5:5ed62483b14663e2a95ebbe115eb8f43',
 466             },
 467             'params': {
 468                 # m3u8 download
 469                 'skip_download': True,
 470             },
 471         },
 472         # TVC embed
 473         {
 474             'url': 'http://sch1298sz.mskobr.ru/dou_edu/karamel_ki/filial_galleries/video/iframe_src_http_tvc_ru_video_iframe_id_55304_isplay_false_acc_video_id_channel_brand_id_11_show_episodes_episode_id_32307_frameb/',
 475             'info_dict': {
 476                 'id': '55304',
 477                 'ext': 'mp4',
 478                 'title': 'Дошкольное воспитание',
 479             },
 480         },
 481         # SportBox embed
 482         {
 483             'url': 'http://www.vestifinance.ru/articles/25753',
 484             'info_dict': {
 485                 'id': '25753',
 486                 'title': 'Прямые трансляции с Форума-выставки "Госзаказ-2013"',
 487             },
 488             'playlist': [{
 489                 'info_dict': {
 490                     'id': '370908',
 491                     'title': 'Госзаказ. День 3',
 492                     'ext': 'mp4',
 493                 }
 494             }, {
 495                 'info_dict': {
 496                     'id': '370905',
 497                     'title': 'Госзаказ. День 2',
 498                     'ext': 'mp4',
 499                 }
 500             }, {
 501                 'info_dict': {
 502                     'id': '370902',
 503                     'title': 'Госзаказ. День 1',
 504                     'ext': 'mp4',
 505                 }
 506             }],
 507             'params': {
 508                 # m3u8 download
 509                 'skip_download': True,
 510             },
 511         },
 512         # Myvi.ru embed
 513         {
 514             'url': 'http://www.kinomyvi.tv/news/detail/Pervij-dublirovannij-trejler--Uzhastikov-_nOw1',
 515             'info_dict': {
 516                 'id': 'f4dafcad-ff21-423d-89b5-146cfd89fa1e',
 517                 'ext': 'mp4',
 518                 'title': 'Ужастики, русский трейлер (2015)',
 519                 'thumbnail': 're:^https?://.*\.jpg$',
 520                 'duration': 153,
 521             }
 522         },
 523         # XHamster embed
 524         {
 525             'url': 'http://www.numisc.com/forum/showthread.php?11696-FM15-which-pumiscer-was-this-%28-vid-%29-%28-alfa-as-fuck-srx-%29&s=711f5db534502e22260dec8c5e2d66d8',
 526             'info_dict': {
 527                 'id': 'showthread',
 528                 'title': '[NSFL] [FM15] which pumiscer was this ( vid ) ( alfa as fuck srx )',
 529             },
 530             'playlist_mincount': 7,
 531             # This forum does not allow <iframe> syntaxes anymore
 532             # Now HTML tags are displayed as-is
 533             'skip': 'No videos on this page',
 534         },
 535         # Embedded TED video
 536         {
 537             'url': 'http://en.support.wordpress.com/videos/ted-talks/',
 538             'md5': '65fdff94098e4a607385a60c5177c638',
 539             'info_dict': {
 540                 'id': '1969',
 541                 'ext': 'mp4',
 542                 'title': 'Hidden miracles of the natural world',
 543                 'uploader': 'Louie Schwartzberg',
 544                 'description': 'md5:8145d19d320ff3e52f28401f4c4283b9',
 545             }
 546         },
 547         # Embedded Ustream video
 548         {
 549             'url': 'http://www.american.edu/spa/pti/nsa-privacy-janus-2014.cfm',
 550             'md5': '27b99cdb639c9b12a79bca876a073417',
 551             'info_dict': {
 552                 'id': '45734260',
 553                 'ext': 'flv',
 554                 'uploader': 'AU SPA:  The NSA and Privacy',
 555                 'title': 'NSA and Privacy Forum Debate featuring General Hayden and Barton Gellman'
 556             }
 557         },
 558         # nowvideo embed hidden behind percent encoding
 559         {
 560             'url': 'http://www.waoanime.tv/the-super-dimension-fortress-macross-episode-1/',
 561             'md5': '2baf4ddd70f697d94b1c18cf796d5107',
 562             'info_dict': {
 563                 'id': '06e53103ca9aa',
 564                 'ext': 'flv',
 565                 'title': 'Macross Episode 001  Watch Macross Episode 001 onl',
 566                 'description': 'No description',
 567             },
 568         },
 569         # arte embed
 570         {
 571             'url': 'http://www.tv-replay.fr/redirection/20-03-14/x-enius-arte-10753389.html',
 572             'md5': '7653032cbb25bf6c80d80f217055fa43',
 573             'info_dict': {
 574                 'id': '048195-004_PLUS7-F',
 575                 'ext': 'flv',
 576                 'title': 'X:enius',
 577                 'description': 'md5:d5fdf32ef6613cdbfd516ae658abf168',
 578                 'upload_date': '20140320',
 579             },
 580             'params': {
 581                 'skip_download': 'Requires rtmpdump'
 582             },
 583             'skip': 'video gone',
 584         },
 585         # francetv embed
 586         {
 587             'url': 'http://www.tsprod.com/replay-du-concert-alcaline-de-calogero',
 588             'info_dict': {
 589                 'id': 'EV_30231',
 590                 'ext': 'mp4',
 591                 'title': 'Alcaline, le concert avec Calogero',
 592                 'description': 'md5:61f08036dcc8f47e9cfc33aed08ffaff',
 593                 'upload_date': '20150226',
 594                 'timestamp': 1424989860,
 595                 'duration': 5400,
 596             },
 597             'params': {
 598                 # m3u8 downloads
 599                 'skip_download': True,
 600             },
 601             'expected_warnings': [
 602                 'Forbidden'
 603             ]
 604         },
 605         # Condé Nast embed
 606         {
 607             'url': 'http://www.wired.com/2014/04/honda-asimo/',
 608             'md5': 'ba0dfe966fa007657bd1443ee672db0f',
 609             'info_dict': {
 610                 'id': '53501be369702d3275860000',
 611                 'ext': 'mp4',
 612                 'title': 'Honda’s  New Asimo Robot Is More Human Than Ever',
 613             }
 614         },
 615         # Dailymotion embed
 616         {
 617             'url': 'http://www.spi0n.com/zap-spi0n-com-n216/',
 618             'md5': '441aeeb82eb72c422c7f14ec533999cd',
 619             'info_dict': {
 620                 'id': 'k2mm4bCdJ6CQ2i7c8o2',
 621                 'ext': 'mp4',
 622                 'title': 'Le Zap de Spi0n n°216 - Zapping du Web',
 623                 'description': 'md5:faf028e48a461b8b7fad38f1e104b119',
 624                 'uploader': 'Spi0n',
 625                 'uploader_id': 'xgditw',
 626                 'upload_date': '20140425',
 627                 'timestamp': 1398441542,
 628             },
 629             'add_ie': ['Dailymotion'],
 630         },
 631         # YouTube embed
 632         {
 633             'url': 'http://www.badzine.de/ansicht/datum/2014/06/09/so-funktioniert-die-neue-englische-badminton-liga.html',
 634             'info_dict': {
 635                 'id': 'FXRb4ykk4S0',
 636                 'ext': 'mp4',
 637                 'title': 'The NBL Auction 2014',
 638                 'uploader': 'BADMINTON England',
 639                 'uploader_id': 'BADMINTONEvents',
 640                 'upload_date': '20140603',
 641                 'description': 'md5:9ef128a69f1e262a700ed83edb163a73',
 642             },
 643             'add_ie': ['Youtube'],
 644             'params': {
 645                 'skip_download': True,
 646             }
 647         },
 648         # MTVSercices embed
 649         {
 650             'url': 'http://www.vulture.com/2016/06/new-key-peele-sketches-released.html',
 651             'md5': 'ca1aef97695ef2c1d6973256a57e5252',
 652             'info_dict': {
 653                 'id': '769f7ec0-0692-4d62-9b45-0d88074bffc1',
 654                 'ext': 'mp4',
 655                 'title': 'Key and Peele|October 10, 2012|2|203|Liam Neesons - Uncensored',
 656                 'description': 'Two valets share their love for movie star Liam Neesons.',
 657                 'timestamp': 1349922600,
 658                 'upload_date': '20121011',
 659             },
 660         },
 661         # YouTube embed via <data-embed-url="">
 662         {
 663             'url': 'https://play.google.com/store/apps/details?id=com.gameloft.android.ANMP.GloftA8HM',
 664             'info_dict': {
 665                 'id': '4vAffPZIT44',
 666                 'ext': 'mp4',
 667                 'title': 'Asphalt 8: Airborne - Update - Welcome to Dubai!',
 668                 'uploader': 'Gameloft',
 669                 'uploader_id': 'gameloft',
 670                 'upload_date': '20140828',
 671                 'description': 'md5:c80da9ed3d83ae6d1876c834de03e1c4',
 672             },
 673             'params': {
 674                 'skip_download': True,
 675             }
 676         },
 677         # Camtasia studio
 678         {
 679             'url': 'http://www.ll.mit.edu/workshops/education/videocourses/antennas/lecture1/video/',
 680             'playlist': [{
 681                 'md5': '0c5e352edabf715d762b0ad4e6d9ee67',
 682                 'info_dict': {
 683                     'id': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final',
 684                     'title': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final - video1',
 685                     'ext': 'flv',
 686                     'duration': 2235.90,
 687                 }
 688             }, {
 689                 'md5': '10e4bb3aaca9fd630e273ff92d9f3c63',
 690                 'info_dict': {
 691                     'id': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final_PIP',
 692                     'title': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final - pip',
 693                     'ext': 'flv',
 694                     'duration': 2235.93,
 695                 }
 696             }],
 697             'info_dict': {
 698                 'title': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final',
 699             }
 700         },
 701         # Flowplayer
 702         {
 703             'url': 'http://www.handjobhub.com/video/busty-blonde-siri-tit-fuck-while-wank-6313.html',
 704             'md5': '9d65602bf31c6e20014319c7d07fba27',
 705             'info_dict': {
 706                 'id': '5123ea6d5e5a7',
 707                 'ext': 'mp4',
 708                 'age_limit': 18,
 709                 'uploader': 'www.handjobhub.com',
 710                 'title': 'Busty Blonde Siri Tit Fuck While Wank at HandjobHub.com',
 711             }
 712         },
 713         # Multiple brightcove videos
 714         # https://github.com/rg3/youtube-dl/issues/2283
 715         {
 716             'url': 'http://www.newyorker.com/online/blogs/newsdesk/2014/01/always-never-nuclear-command-and-control.html',
 717             'info_dict': {
 718                 'id': 'always-never',
 719                 'title': 'Always / Never - The New Yorker',
 720             },
 721             'playlist_count': 3,
 722             'params': {
 723                 'extract_flat': False,
 724                 'skip_download': True,
 725             }
 726         },
 727         # MLB embed
 728         {
 729             'url': 'http://umpire-empire.com/index.php/topic/58125-laz-decides-no-thats-low/',
 730             'md5': '96f09a37e44da40dd083e12d9a683327',
 731             'info_dict': {
 732                 'id': '33322633',
 733                 'ext': 'mp4',
 734                 'title': 'Ump changes call to ball',
 735                 'description': 'md5:71c11215384298a172a6dcb4c2e20685',
 736                 'duration': 48,
 737                 'timestamp': 1401537900,
 738                 'upload_date': '20140531',
 739                 'thumbnail': 're:^https?://.*\.jpg$',
 740             },
 741         },
 742         # Wistia embed
 743         {
 744             'url': 'http://study.com/academy/lesson/north-american-exploration-failed-colonies-of-spain-france-england.html#lesson',
 745             'md5': '1953f3a698ab51cfc948ed3992a0b7ff',
 746             'info_dict': {
 747                 'id': '6e2wtrbdaf',
 748                 'ext': 'mov',
 749                 'title': 'paywall_north-american-exploration-failed-colonies-of-spain-france-england',
 750                 'description': 'a Paywall Videos video from Remilon',
 751                 'duration': 644.072,
 752                 'uploader': 'study.com',
 753                 'timestamp': 1459678540,
 754                 'upload_date': '20160403',
 755                 'filesize': 24687186,
 756             },
 757         },
 758         {
 759             'url': 'http://thoughtworks.wistia.com/medias/uxjb0lwrcz',
 760             'md5': 'baf49c2baa8a7de5f3fc145a8506dcd4',
 761             'info_dict': {
 762                 'id': 'uxjb0lwrcz',
 763                 'ext': 'mp4',
 764                 'title': 'Conversation about Hexagonal Rails Part 1',
 765                 'description': 'a Martin Fowler video from ThoughtWorks',
 766                 'duration': 1715.0,
 767                 'uploader': 'thoughtworks.wistia.com',
 768                 'timestamp': 1401832161,
 769                 'upload_date': '20140603',
 770             },
 771         },
 772         # Wistia standard embed (async)
 773         {
 774             'url': 'https://www.getdrip.com/university/brennan-dunn-drip-workshop/',
 775             'info_dict': {
 776                 'id': '807fafadvk',
 777                 'ext': 'mp4',
 778                 'title': 'Drip Brennan Dunn Workshop',
 779                 'description': 'a JV Webinars video from getdrip-1',
 780                 'duration': 4986.95,
 781                 'timestamp': 1463607249,
 782                 'upload_date': '20160518',
 783             },
 784             'params': {
 785                 'skip_download': True,
 786             }
 787         },
 788         # Soundcloud embed
 789         {
 790             'url': 'http://nakedsecurity.sophos.com/2014/10/29/sscc-171-are-you-sure-that-1234-is-a-bad-password-podcast/',
 791             'info_dict': {
 792                 'id': '174391317',
 793                 'ext': 'mp3',
 794                 'description': 'md5:ff867d6b555488ad3c52572bb33d432c',
 795                 'uploader': 'Sophos Security',
 796                 'title': 'Chet Chat 171 - Oct 29, 2014',
 797                 'upload_date': '20141029',
 798             }
 799         },
 800         # Soundcloud multiple embeds
 801         {
 802             'url': 'http://www.guitarplayer.com/lessons/1014/legato-workout-one-hour-to-more-fluid-performance---tab/52809',
 803             'info_dict': {
 804                 'id': '52809',
 805                 'title': 'Guitar Essentials: Legato Workout—One-Hour to Fluid Performance  | TAB + AUDIO',
 806             },
 807             'playlist_mincount': 7,
 808         },
 809         # Livestream embed
 810         {
 811             'url': 'http://www.esa.int/Our_Activities/Space_Science/Rosetta/Philae_comet_touch-down_webcast',
 812             'info_dict': {
 813                 'id': '67864563',
 814                 'ext': 'flv',
 815                 'upload_date': '20141112',
 816                 'title': 'Rosetta #CometLanding webcast HL 10',
 817             }
 818         },
 819         # Another Livestream embed, without 'new.' in URL
 820         {
 821             'url': 'https://www.freespeech.org/',
 822             'info_dict': {
 823                 'id': '123537347',
 824                 'ext': 'mp4',
 825                 'title': 're:^FSTV [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
 826             },
 827             'params': {
 828                 # Live stream
 829                 'skip_download': True,
 830             },
 831         },
 832         # LazyYT
 833         {
 834             'url': 'http://discourse.ubuntu.com/t/unity-8-desktop-mode-windows-on-mir/1986',
 835             'info_dict': {
 836                 'id': '1986',
 837                 'title': 'Unity 8 desktop-mode windows on Mir! - Ubuntu Discourse',
 838             },
 839             'playlist_mincount': 2,
 840         },
 841         # Cinchcast embed
 842         {
 843             'url': 'http://undergroundwellness.com/podcasts/306-5-steps-to-permanent-gut-healing/',
 844             'info_dict': {
 845                 'id': '7141703',
 846                 'ext': 'mp3',
 847                 'upload_date': '20141126',
 848                 'title': 'Jack Tips: 5 Steps to Permanent Gut Healing',
 849             }
 850         },
 851         # Cinerama player
 852         {
 853             'url': 'http://www.abc.net.au/7.30/content/2015/s4164797.htm',
 854             'info_dict': {
 855                 'id': '730m_DandD_1901_512k',
 856                 'ext': 'mp4',
 857                 'uploader': 'www.abc.net.au',
 858                 'title': 'Game of Thrones with dice - Dungeons and Dragons fantasy role-playing game gets new life - 19/01/2015',
 859             }
 860         },
 861         # embedded viddler video
 862         {
 863             'url': 'http://deadspin.com/i-cant-stop-watching-john-wall-chop-the-nuggets-with-th-1681801597',
 864             'info_dict': {
 865                 'id': '4d03aad9',
 866                 'ext': 'mp4',
 867                 'uploader': 'deadspin',
 868                 'title': 'WALL-TO-GORTAT',
 869                 'timestamp': 1422285291,
 870                 'upload_date': '20150126',
 871             },
 872             'add_ie': ['Viddler'],
 873         },
 874         # Libsyn embed
 875         {
 876             'url': 'http://thedailyshow.cc.com/podcast/episodetwelve',
 877             'info_dict': {
 878                 'id': '3377616',
 879                 'ext': 'mp3',
 880                 'title': "The Daily Show Podcast without Jon Stewart - Episode 12: Bassem Youssef: Egypt's Jon Stewart",
 881                 'description': 'md5:601cb790edd05908957dae8aaa866465',
 882                 'upload_date': '20150220',
 883             },
 884             'skip': 'All The Daily Show URLs now redirect to http://www.cc.com/shows/',
 885         },
 886         # jwplayer YouTube
 887         {
 888             'url': 'http://media.nationalarchives.gov.uk/index.php/webinar-using-discovery-national-archives-online-catalogue/',
 889             'info_dict': {
 890                 'id': 'Mrj4DVp2zeA',
 891                 'ext': 'mp4',
 892                 'upload_date': '20150212',
 893                 'uploader': 'The National Archives UK',
 894                 'description': 'md5:a236581cd2449dd2df4f93412f3f01c6',
 895                 'uploader_id': 'NationalArchives08',
 896                 'title': 'Webinar: Using Discovery, The National Archives’ online catalogue',
 897             },
 898         },
 899         # rtl.nl embed
 900         {
 901             'url': 'http://www.rtlnieuws.nl/nieuws/buitenland/aanslagen-kopenhagen',
 902             'playlist_mincount': 5,
 903             'info_dict': {
 904                 'id': 'aanslagen-kopenhagen',
 905                 'title': 'Aanslagen Kopenhagen | RTL Nieuws',
 906             }
 907         },
 908         # Zapiks embed
 909         {
 910             'url': 'http://www.skipass.com/news/116090-bon-appetit-s5ep3-baqueira-mi-cor.html',
 911             'info_dict': {
 912                 'id': '118046',
 913                 'ext': 'mp4',
 914                 'title': 'EP3S5 - Bon Appétit - Baqueira Mi Corazon !',
 915             }
 916         },
 917         # Kaltura embed (different embed code)
 918         {
 919             'url': 'http://www.premierchristianradio.com/Shows/Saturday/Unbelievable/Conference-Videos/Os-Guinness-Is-It-Fools-Talk-Unbelievable-Conference-2014',
 920             'info_dict': {
 921                 'id': '1_a52wc67y',
 922                 'ext': 'flv',
 923                 'upload_date': '20150127',
 924                 'uploader_id': 'PremierMedia',
 925                 'timestamp': int,
 926                 'title': 'Os Guinness // Is It Fools Talk? // Unbelievable? Conference 2014',
 927             },
 928         },
 929         # Kaltura embed protected with referrer
 930         {
 931             'url': 'http://www.disney.nl/disney-channel/filmpjes/achter-de-schermen#/videoId/violetta-achter-de-schermen-ruggero',
 932             'info_dict': {
 933                 'id': '1_g4fbemnq',
 934                 'ext': 'mp4',
 935                 'title': 'Violetta - Achter De Schermen - Ruggero',
 936                 'description': 'Achter de schermen met Ruggero',
 937                 'timestamp': 1435133761,
 938                 'upload_date': '20150624',
 939                 'uploader_id': 'echojecka',
 940             },
 941         },
 942         # Kaltura embed with single quotes
 943         {
 944             'url': 'http://fod.infobase.com/p_ViewPlaylist.aspx?AssignmentID=NUN8ZY',
 945             'info_dict': {
 946                 'id': '0_izeg5utt',
 947                 'ext': 'mp4',
 948                 'title': '35871',
 949                 'timestamp': 1355743100,
 950                 'upload_date': '20121217',
 951                 'uploader_id': 'batchUser',
 952             },
 953             'add_ie': ['Kaltura'],
 954         },
 955         {
 956             # Kaltura embedded via quoted entry_id
 957             'url': 'https://www.oreilly.com/ideas/my-cloud-makes-pretty-pictures',
 958             'info_dict': {
 959                 'id': '0_utuok90b',
 960                 'ext': 'mp4',
 961                 'title': '06_matthew_brender_raj_dutt',
 962                 'timestamp': 1466638791,
 963                 'upload_date': '20160622',
 964             },
 965             'add_ie': ['Kaltura'],
 966             'expected_warnings': [
 967                 'Could not send HEAD request'
 968             ],
 969             'params': {
 970                 'skip_download': True,
 971             }
 972         },
 973         # Eagle.Platform embed (generic URL)
 974         {
 975             'url': 'http://lenta.ru/news/2015/03/06/navalny/',
 976             # Not checking MD5 as sometimes the direct HTTP link results in 404 and HLS is used
 977             'info_dict': {
 978                 'id': '227304',
 979                 'ext': 'mp4',
 980                 'title': 'Навальный вышел на свободу',
 981                 'description': 'md5:d97861ac9ae77377f3f20eaf9d04b4f5',
 982                 'thumbnail': 're:^https?://.*\.jpg$',
 983                 'duration': 87,
 984                 'view_count': int,
 985                 'age_limit': 0,
 986             },
 987         },
 988         # ClipYou (Eagle.Platform) embed (custom URL)
 989         {
 990             'url': 'http://muz-tv.ru/play/7129/',
 991             # Not checking MD5 as sometimes the direct HTTP link results in 404 and HLS is used
 992             'info_dict': {
 993                 'id': '12820',
 994                 'ext': 'mp4',
 995                 'title': "'O Sole Mio",
 996                 'thumbnail': 're:^https?://.*\.jpg$',
 997                 'duration': 216,
 998                 'view_count': int,
 999             },
1000         },
1001         # Pladform embed
1002         {
1003             'url': 'http://muz-tv.ru/kinozal/view/7400/',
1004             'info_dict': {
1005                 'id': '100183293',
1006                 'ext': 'mp4',
1007                 'title': 'Тайны перевала Дятлова • 1 серия 2 часть',
1008                 'description': 'Документальный сериал-расследование одной из самых жутких тайн ХХ века',
1009                 'thumbnail': 're:^https?://.*\.jpg$',
1010                 'duration': 694,
1011                 'age_limit': 0,
1012             },
1013         },
1014         # Playwire embed
1015         {
1016             'url': 'http://www.cinemablend.com/new/First-Joe-Dirt-2-Trailer-Teaser-Stupid-Greatness-70874.html',
1017             'info_dict': {
1018                 'id': '3519514',
1019                 'ext': 'mp4',
1020                 'title': 'Joe Dirt 2 Beautiful Loser Teaser Trailer',
1021                 'thumbnail': 're:^https?://.*\.png$',
1022                 'duration': 45.115,
1023             },
1024         },
1025         # 5min embed
1026         {
1027             'url': 'http://techcrunch.com/video/facebook-creates-on-this-day-crunch-report/518726732/',
1028             'md5': '4c6f127a30736b59b3e2c19234ee2bf7',
1029             'info_dict': {
1030                 'id': '518726732',
1031                 'ext': 'mp4',
1032                 'title': 'Facebook Creates "On This Day" | Crunch Report',
1033             },
1034         },
1035         # SVT embed
1036         {
1037             'url': 'http://www.svt.se/sport/ishockey/jagr-tacklar-giroux-under-intervjun',
1038             'info_dict': {
1039                 'id': '2900353',
1040                 'ext': 'flv',
1041                 'title': 'Här trycker Jagr till Giroux (under SVT-intervjun)',
1042                 'duration': 27,
1043                 'age_limit': 0,
1044             },
1045         },
1046         # Crooks and Liars embed
1047         {
1048             'url': 'http://crooksandliars.com/2015/04/fox-friends-says-protecting-atheists',
1049             'info_dict': {
1050                 'id': '8RUoRhRi',
1051                 'ext': 'mp4',
1052                 'title': "Fox & Friends Says Protecting Atheists From Discrimination Is Anti-Christian!",
1053                 'description': 'md5:e1a46ad1650e3a5ec7196d432799127f',
1054                 'timestamp': 1428207000,
1055                 'upload_date': '20150405',
1056                 'uploader': 'Heather',
1057             },
1058         },
1059         # Crooks and Liars external embed
1060         {
1061             'url': 'http://theothermccain.com/2010/02/02/video-proves-that-bill-kristol-has-been-watching-glenn-beck/comment-page-1/',
1062             'info_dict': {
1063                 'id': 'MTE3MjUtMzQ2MzA',
1064                 'ext': 'mp4',
1065                 'title': 'md5:5e3662a81a4014d24c250d76d41a08d5',
1066                 'description': 'md5:9b8e9542d6c3c5de42d6451b7d780cec',
1067                 'timestamp': 1265032391,
1068                 'upload_date': '20100201',
1069                 'uploader': 'Heather',
1070             },
1071         },
1072         # NBC Sports vplayer embed
1073         {
1074             'url': 'http://www.riderfans.com/forum/showthread.php?121827-Freeman&s=e98fa1ea6dc08e886b1678d35212494a',
1075             'info_dict': {
1076                 'id': 'ln7x1qSThw4k',
1077                 'ext': 'flv',
1078                 'title': "PFT Live: New leader in the 'new-look' defense",
1079                 'description': 'md5:65a19b4bbfb3b0c0c5768bed1dfad74e',
1080                 'uploader': 'NBCU-SPORTS',
1081                 'upload_date': '20140107',
1082                 'timestamp': 1389118457,
1083             },
1084         },
1085         # NBC News embed
1086         {
1087             'url': 'http://www.vulture.com/2016/06/letterman-couldnt-care-less-about-late-night.html',
1088             'md5': '1aa589c675898ae6d37a17913cf68d66',
1089             'info_dict': {
1090                 'id': '701714499682',
1091                 'ext': 'mp4',
1092                 'title': 'PREVIEW: On Assignment: David Letterman',
1093                 'description': 'A preview of Tom Brokaw\'s interview with David Letterman as part of the On Assignment series powered by Dateline. Airs Sunday June 12 at 7/6c.',
1094             },
1095         },
1096         # UDN embed
1097         {
1098             'url': 'https://video.udn.com/news/300346',
1099             'md5': 'fd2060e988c326991037b9aff9df21a6',
1100             'info_dict': {
1101                 'id': '300346',
1102                 'ext': 'mp4',
1103                 'title': '中一中男師變性 全校師生力挺',
1104                 'thumbnail': 're:^https?://.*\.jpg$',
1105             },
1106             'params': {
1107                 # m3u8 download
1108                 'skip_download': True,
1109             },
1110         },
1111         # Ooyala embed
1112         {
1113             'url': 'http://www.businessinsider.com/excel-index-match-vlookup-video-how-to-2015-2?IR=T',
1114             'info_dict': {
1115                 'id': '50YnY4czr4ms1vJ7yz3xzq0excz_pUMs',
1116                 'ext': 'mp4',
1117                 'description': 'VIDEO: INDEX/MATCH versus VLOOKUP.',
1118                 'title': 'This is what separates the Excel masters from the wannabes',
1119                 'duration': 191.933,
1120             },
1121             'params': {
1122                 # m3u8 downloads
1123                 'skip_download': True,
1124             }
1125         },
1126         # Brightcove URL in single quotes
1127         {
1128             'url': 'http://www.sportsnet.ca/baseball/mlb/sn-presents-russell-martin-world-citizen/',
1129             'md5': '4ae374f1f8b91c889c4b9203c8c752af',
1130             'info_dict': {
1131                 'id': '4255764656001',
1132                 'ext': 'mp4',
1133                 'title': 'SN Presents: Russell Martin, World Citizen',
1134                 'description': 'To understand why he was the Toronto Blue Jays’ top off-season priority is to appreciate his background and upbringing in Montreal, where he first developed his baseball skills. Written and narrated by Stephen Brunt.',
1135                 'uploader': 'Rogers Sportsnet',
1136                 'uploader_id': '1704050871',
1137                 'upload_date': '20150525',
1138                 'timestamp': 1432570283,
1139             },
1140         },
1141         # Dailymotion Cloud video
1142         {
1143             'url': 'http://replay.publicsenat.fr/vod/le-debat/florent-kolandjian,dominique-cena,axel-decourtye,laurence-abeille,bruno-parmentier/175910',
1144             'md5': 'dcaf23ad0c67a256f4278bce6e0bae38',
1145             'info_dict': {
1146                 'id': 'x2uy8t3',
1147                 'ext': 'mp4',
1148                 'title': 'Sauvons les abeilles ! - Le débat',
1149                 'description': 'md5:d9082128b1c5277987825d684939ca26',
1150                 'thumbnail': 're:^https?://.*\.jpe?g$',
1151                 'timestamp': 1434970506,
1152                 'upload_date': '20150622',
1153                 'uploader': 'Public Sénat',
1154                 'uploader_id': 'xa9gza',
1155             }
1156         },
1157         # OnionStudios embed
1158         {
1159             'url': 'http://www.clickhole.com/video/dont-understand-bitcoin-man-will-mumble-explanatio-2537',
1160             'info_dict': {
1161                 'id': '2855',
1162                 'ext': 'mp4',
1163                 'title': 'Don’t Understand Bitcoin? This Man Will Mumble An Explanation At You',
1164                 'thumbnail': 're:^https?://.*\.jpe?g$',
1165                 'uploader': 'ClickHole',
1166                 'uploader_id': 'clickhole',
1167             }
1168         },
1169         # SnagFilms embed
1170         {
1171             'url': 'http://whilewewatch.blogspot.ru/2012/06/whilewewatch-whilewewatch-gripping.html',
1172             'info_dict': {
1173                 'id': '74849a00-85a9-11e1-9660-123139220831',
1174                 'ext': 'mp4',
1175                 'title': '#whilewewatch',
1176             }
1177         },
1178         # AdobeTVVideo embed
1179         {
1180             'url': 'https://helpx.adobe.com/acrobat/how-to/new-experience-acrobat-dc.html?set=acrobat--get-started--essential-beginners',
1181             'md5': '43662b577c018ad707a63766462b1e87',
1182             'info_dict': {
1183                 'id': '2456',
1184                 'ext': 'mp4',
1185                 'title': 'New experience with Acrobat DC',
1186                 'description': 'New experience with Acrobat DC',
1187                 'duration': 248.667,
1188             },
1189         },
1190         # ScreenwaveMedia embed
1191         {
1192             'url': 'http://www.thecinemasnob.com/the-cinema-snob/a-nightmare-on-elm-street-2-freddys-revenge1',
1193             'md5': '24ace5baba0d35d55c6810b51f34e9e0',
1194             'info_dict': {
1195                 'id': 'cinemasnob-55d26273809dd',
1196                 'ext': 'mp4',
1197                 'title': 'cinemasnob',
1198             },
1199         },
1200         # BrightcoveInPageEmbed embed
1201         {
1202             'url': 'http://www.geekandsundry.com/tabletop-bonus-wils-final-thoughts-on-dread/',
1203             'info_dict': {
1204                 'id': '4238694884001',
1205                 'ext': 'flv',
1206                 'title': 'Tabletop: Dread, Last Thoughts',
1207                 'description': 'Tabletop: Dread, Last Thoughts',
1208                 'duration': 51690,
1209             },
1210         },
1211         # JWPlayer with M3U8
1212         {
1213             'url': 'http://ren.tv/novosti/2015-09-25/sluchaynyy-prohozhiy-poymal-avtougonshchika-v-murmanske-video',
1214             'info_dict': {
1215                 'id': 'playlist',
1216                 'ext': 'mp4',
1217                 'title': 'Случайный прохожий поймал автоугонщика в Мурманске. ВИДЕО | РЕН ТВ',
1218                 'uploader': 'ren.tv',
1219             },
1220             'params': {
1221                 # m3u8 downloads
1222                 'skip_download': True,
1223             }
1224         },
1225         # Brightcove embed, with no valid 'renditions' but valid 'IOSRenditions'
1226         # This video can't be played in browsers if Flash disabled and UA set to iPhone, which is actually a false alarm
1227         {
1228             'url': 'https://dl.dropboxusercontent.com/u/29092637/interview.html',
1229             'info_dict': {
1230                 'id': '4785848093001',
1231                 'ext': 'mp4',
1232                 'title': 'The Cardinal Pell Interview',
1233                 'description': 'Sky News Contributor Andrew Bolt interviews George Pell in Rome, following the Cardinal\'s evidence before the Royal Commission into Child Abuse. ',
1234                 'uploader': 'GlobeCast Australia - GlobeStream',
1235                 'uploader_id': '2733773828001',
1236                 'upload_date': '20160304',
1237                 'timestamp': 1457083087,
1238             },
1239             'params': {
1240                 # m3u8 downloads
1241                 'skip_download': True,
1242             },
1243         },
1244         # Another form of arte.tv embed
1245         {
1246             'url': 'http://www.tv-replay.fr/redirection/09-04-16/arte-reportage-arte-11508975.html',
1247             'md5': '850bfe45417ddf221288c88a0cffe2e2',
1248             'info_dict': {
1249                 'id': '030273-562_PLUS7-F',
1250                 'ext': 'mp4',
1251                 'title': 'ARTE Reportage - Nulle part, en France',
1252                 'description': 'md5:e3a0e8868ed7303ed509b9e3af2b870d',
1253                 'upload_date': '20160409',
1254             },
1255         },
1256         # LiveLeak embed
1257         {
1258             'url': 'http://www.wykop.pl/link/3088787/',
1259             'md5': 'ace83b9ed19b21f68e1b50e844fdf95d',
1260             'info_dict': {
1261                 'id': '874_1459135191',
1262                 'ext': 'mp4',
1263                 'title': 'Man shows poor quality of new apartment building',
1264                 'description': 'The wall is like a sand pile.',
1265                 'uploader': 'Lake8737',
1266             }
1267         },
1268         # Duplicated embedded video URLs
1269         {
1270             'url': 'http://www.hudl.com/athlete/2538180/highlights/149298443',
1271             'info_dict': {
1272                 'id': '149298443_480_16c25b74_2',
1273                 'ext': 'mp4',
1274                 'title': 'vs. Blue Orange Spring Game',
1275                 'uploader': 'www.hudl.com',
1276             },
1277         },
1278         # twitter:player:stream embed
1279         {
1280             'url': 'http://www.rtl.be/info/video/589263.aspx?CategoryID=288',
1281             'info_dict': {
1282                 'id': 'master',
1283                 'ext': 'mp4',
1284                 'title': 'Une nouvelle espèce de dinosaure découverte en Argentine',
1285                 'uploader': 'www.rtl.be',
1286             },
1287             'params': {
1288                 # m3u8 downloads
1289                 'skip_download': True,
1290             },
1291         },
1292         # twitter:player embed
1293         {
1294             'url': 'http://www.theatlantic.com/video/index/484130/what-do-black-holes-sound-like/',
1295             'md5': 'a3e0df96369831de324f0778e126653c',
1296             'info_dict': {
1297                 'id': '4909620399001',
1298                 'ext': 'mp4',
1299                 'title': 'What Do Black Holes Sound Like?',
1300                 'description': 'what do black holes sound like',
1301                 'upload_date': '20160524',
1302                 'uploader_id': '29913724001',
1303                 'timestamp': 1464107587,
1304                 'uploader': 'TheAtlantic',
1305             },
1306             'add_ie': ['BrightcoveLegacy'],
1307         },
1308         # Facebook <iframe> embed
1309         {
1310             'url': 'https://www.hostblogger.de/blog/archives/6181-Auto-jagt-Betonmischer.html',
1311             'md5': 'fbcde74f534176ecb015849146dd3aee',
1312             'info_dict': {
1313                 'id': '599637780109885',
1314                 'ext': 'mp4',
1315                 'title': 'Facebook video #599637780109885',
1316             },
1317         },
1318         # Facebook API embed
1319         {
1320             'url': 'http://www.lothype.com/blue-stars-2016-preview-standstill-full-show/',
1321             'md5': 'a47372ee61b39a7b90287094d447d94e',
1322             'info_dict': {
1323                 'id': '10153467542406923',
1324                 'ext': 'mp4',
1325                 'title': 'Facebook video #10153467542406923',
1326             },
1327         },
1328         # Wordpress "YouTube Video Importer" plugin
1329         {
1330             'url': 'http://www.lothype.com/blue-devils-drumline-stanford-lot-2016/',
1331             'md5': 'd16797741b560b485194eddda8121b48',
1332             'info_dict': {
1333                 'id': 'HNTXWDXV9Is',
1334                 'ext': 'mp4',
1335                 'title': 'Blue Devils Drumline Stanford lot 2016',
1336                 'upload_date': '20160627',
1337                 'uploader_id': 'GENOCIDE8GENERAL10',
1338                 'uploader': 'cylus cyrus',
1339             },
1340         },
1341         {
1342             # video stored on custom kaltura server
1343             'url': 'http://www.expansion.com/multimedia/videos.html?media=EQcM30NHIPv',
1344             'md5': '537617d06e64dfed891fa1593c4b30cc',
1345             'info_dict': {
1346                 'id': '0_1iotm5bh',
1347                 'ext': 'mp4',
1348                 'title': 'Elecciones británicas: 5 lecciones para Rajoy',
1349                 'description': 'md5:435a89d68b9760b92ce67ed227055f16',
1350                 'uploader_id': 'videos.expansion@el-mundo.net',
1351                 'upload_date': '20150429',
1352                 'timestamp': 1430303472,
1353             },
1354             'add_ie': ['Kaltura'],
1355         },
1356         {
1357             # Non-standard Vimeo embed
1358             'url': 'https://openclassrooms.com/courses/understanding-the-web',
1359             'md5': '64d86f1c7d369afd9a78b38cbb88d80a',
1360             'info_dict': {
1361                 'id': '148867247',
1362                 'ext': 'mp4',
1363                 'title': 'Understanding the web - Teaser',
1364                 'description': 'This is "Understanding the web - Teaser" by openclassrooms on Vimeo, the home for high quality videos and the people who love them.',
1365                 'upload_date': '20151214',
1366                 'uploader': 'OpenClassrooms',
1367                 'uploader_id': 'openclassrooms',
1368             },
1369             'add_ie': ['Vimeo'],
1370         },
1371         {
1372             # generic vimeo embed that requires original URL passed as Referer
1373             'url': 'http://racing4everyone.eu/2016/07/30/formula-1-2016-round12-germany/',
1374             'only_matching': True,
1375         },
1376         {
1377             'url': 'https://support.arkena.com/display/PLAY/Ways+to+embed+your+video',
1378             'md5': 'b96f2f71b359a8ecd05ce4e1daa72365',
1379             'info_dict': {
1380                 'id': 'b41dda37-d8e7-4d3f-b1b5-9a9db578bdfe',
1381                 'ext': 'mp4',
1382                 'title': 'Big Buck Bunny',
1383                 'description': 'Royalty free test video',
1384                 'timestamp': 1432816365,
1385                 'upload_date': '20150528',
1386                 'is_live': False,
1387             },
1388             'params': {
1389                 'skip_download': True,
1390             },
1391             'add_ie': [ArkenaIE.ie_key()],
1392         },
1393         {
1394             'url': 'http://nova.bg/news/view/2016/08/16/156543/%D0%BD%D0%B0-%D0%BA%D0%BE%D1%81%D1%8A%D0%BC-%D0%BE%D1%82-%D0%B2%D0%B7%D1%80%D0%B8%D0%B2-%D0%BE%D1%82%D1%86%D0%B5%D0%BF%D0%B8%D1%85%D0%B0-%D1%86%D1%8F%D0%BB-%D0%BA%D0%B2%D0%B0%D1%80%D1%82%D0%B0%D0%BB-%D0%B7%D0%B0%D1%80%D0%B0%D0%B4%D0%B8-%D0%B8%D0%B7%D1%82%D0%B8%D1%87%D0%B0%D0%BD%D0%B5-%D0%BD%D0%B0-%D0%B3%D0%B0%D0%B7-%D0%B2-%D0%BF%D0%BB%D0%BE%D0%B2%D0%B4%D0%B8%D0%B2/',
1395             'info_dict': {
1396                 'id': '1c7141f46c',
1397                 'ext': 'mp4',
1398                 'title': 'НА КОСЪМ ОТ ВЗРИВ: Изтичане на газ на бензиностанция в Пловдив',
1399             },
1400             'params': {
1401                 'skip_download': True,
1402             },
1403             'add_ie': [Vbox7IE.ie_key()],
1404         },
1405         {
1406             # DBTV embeds
1407             'url': 'http://www.dagbladet.no/2016/02/23/nyheter/nordlys/ski/troms/ver/43254897/',
1408             'info_dict': {
1409                 'id': '43254897',
1410                 'title': 'Etter ett års planlegging, klaffet endelig alt: - Jeg måtte ta en liten dans',
1411             },
1412             'playlist_mincount': 3,
1413         },
1414         # {
1415         #     # TODO: find another test
1416         #     # http://schema.org/VideoObject
1417         #     'url': 'https://flipagram.com/f/nyvTSJMKId',
1418         #     'md5': '888dcf08b7ea671381f00fab74692755',
1419         #     'info_dict': {
1420         #         'id': 'nyvTSJMKId',
1421         #         'ext': 'mp4',
1422         #         'title': 'Flipagram by sjuria101 featuring Midnight Memories by One Direction',
1423         #         'description': '#love for cats.',
1424         #         'timestamp': 1461244995,
1425         #         'upload_date': '20160421',
1426         #     },
1427         #     'params': {
1428         #         'force_generic_extractor': True,
1429         #     },
1430         # }
1431     ]
1432
1433     def report_following_redirect(self, new_url):
1434         """Report information extraction."""
1435         self._downloader.to_screen('[redirect] Following redirect to %s' % new_url)
1436
1437     def _extract_rss(self, url, video_id, doc):
1438         playlist_title = doc.find('./channel/title').text
1439         playlist_desc_el = doc.find('./channel/description')
1440         playlist_desc = None if playlist_desc_el is None else playlist_desc_el.text
1441
1442         entries = []
1443         for it in doc.findall('./channel/item'):
1444             next_url = xpath_text(it, 'link', fatal=False)
1445             if not next_url:
1446                 enclosure_nodes = it.findall('./enclosure')
1447                 for e in enclosure_nodes:
1448                     next_url = e.attrib.get('url')
1449                     if next_url:
1450                         break
1451
1452             if not next_url:
1453                 continue
1454
1455             entries.append({
1456                 '_type': 'url',
1457                 'url': next_url,
1458                 'title': it.find('title').text,
1459             })
1460
1461         return {
1462             '_type': 'playlist',
1463             'id': url,
1464             'title': playlist_title,
1465             'description': playlist_desc,
1466             'entries': entries,
1467         }
1468
1469     def _extract_camtasia(self, url, video_id, webpage):
1470         """ Returns None if no camtasia video can be found. """
1471
1472         camtasia_cfg = self._search_regex(
1473             r'fo\.addVariable\(\s*"csConfigFile",\s*"([^"]+)"\s*\);',
1474             webpage, 'camtasia configuration file', default=None)
1475         if camtasia_cfg is None:
1476             return None
1477
1478         title = self._html_search_meta('DC.title', webpage, fatal=True)
1479
1480         camtasia_url = compat_urlparse.urljoin(url, camtasia_cfg)
1481         camtasia_cfg = self._download_xml(
1482             camtasia_url, video_id,
1483             note='Downloading camtasia configuration',
1484             errnote='Failed to download camtasia configuration')
1485         fileset_node = camtasia_cfg.find('./playlist/array/fileset')
1486
1487         entries = []
1488         for n in fileset_node.getchildren():
1489             url_n = n.find('./uri')
1490             if url_n is None:
1491                 continue
1492
1493             entries.append({
1494                 'id': os.path.splitext(url_n.text.rpartition('/')[2])[0],
1495                 'title': '%s - %s' % (title, n.tag),
1496                 'url': compat_urlparse.urljoin(url, url_n.text),
1497                 'duration': float_or_none(n.find('./duration').text),
1498             })
1499
1500         return {
1501             '_type': 'playlist',
1502             'entries': entries,
1503             'title': title,
1504         }
1505
1506     def _real_extract(self, url):
1507         if url.startswith('//'):
1508             return {
1509                 '_type': 'url',
1510                 'url': self.http_scheme() + url,
1511             }
1512
1513         parsed_url = compat_urlparse.urlparse(url)
1514         if not parsed_url.scheme:
1515             default_search = self._downloader.params.get('default_search')
1516             if default_search is None:
1517                 default_search = 'fixup_error'
1518
1519             if default_search in ('auto', 'auto_warning', 'fixup_error'):
1520                 if '/' in url:
1521                     self._downloader.report_warning('The url doesn\'t specify the protocol, trying with http')
1522                     return self.url_result('http://' + url)
1523                 elif default_search != 'fixup_error':
1524                     if default_search == 'auto_warning':
1525                         if re.match(r'^(?:url|URL)$', url):
1526                             raise ExtractorError(
1527                                 'Invalid URL:  %r . Call youtube-dl like this:  youtube-dl -v "https://www.youtube.com/watch?v=BaW_jenozKc"  ' % url,
1528                                 expected=True)
1529                         else:
1530                             self._downloader.report_warning(
1531                                 'Falling back to youtube search for  %s . Set --default-search "auto" to suppress this warning.' % url)
1532                     return self.url_result('ytsearch:' + url)
1533
1534             if default_search in ('error', 'fixup_error'):
1535                 raise ExtractorError(
1536                     '%r is not a valid URL. '
1537                     'Set --default-search "ytsearch" (or run  youtube-dl "ytsearch:%s" ) to search YouTube'
1538                     % (url, url), expected=True)
1539             else:
1540                 if ':' not in default_search:
1541                     default_search += ':'
1542                 return self.url_result(default_search + url)
1543
1544         url, smuggled_data = unsmuggle_url(url)
1545         force_videoid = None
1546         is_intentional = smuggled_data and smuggled_data.get('to_generic')
1547         if smuggled_data and 'force_videoid' in smuggled_data:
1548             force_videoid = smuggled_data['force_videoid']
1549             video_id = force_videoid
1550         else:
1551             video_id = self._generic_id(url)
1552
1553         self.to_screen('%s: Requesting header' % video_id)
1554
1555         head_req = HEADRequest(url)
1556         head_response = self._request_webpage(
1557             head_req, video_id,
1558             note=False, errnote='Could not send HEAD request to %s' % url,
1559             fatal=False)
1560
1561         if head_response is not False:
1562             # Check for redirect
1563             new_url = head_response.geturl()
1564             if url != new_url:
1565                 self.report_following_redirect(new_url)
1566                 if force_videoid:
1567                     new_url = smuggle_url(
1568                         new_url, {'force_videoid': force_videoid})
1569                 return self.url_result(new_url)
1570
1571         full_response = None
1572         if head_response is False:
1573             request = sanitized_Request(url)
1574             request.add_header('Accept-Encoding', '*')
1575             full_response = self._request_webpage(request, video_id)
1576             head_response = full_response
1577
1578         info_dict = {
1579             'id': video_id,
1580             'title': self._generic_title(url),
1581             'upload_date': unified_strdate(head_response.headers.get('Last-Modified'))
1582         }
1583
1584         # Check for direct link to a video
1585         content_type = head_response.headers.get('Content-Type', '').lower()
1586         m = re.match(r'^(?P<type>audio|video|application(?=/(?:ogg$|(?:vnd\.apple\.|x-)?mpegurl)))/(?P<format_id>[^;\s]+)', content_type)
1587         if m:
1588             format_id = m.group('format_id')
1589             if format_id.endswith('mpegurl'):
1590                 formats = self._extract_m3u8_formats(url, video_id, 'mp4')
1591             elif format_id == 'f4m':
1592                 formats = self._extract_f4m_formats(url, video_id)
1593             else:
1594                 formats = [{
1595                     'format_id': m.group('format_id'),
1596                     'url': url,
1597                     'vcodec': 'none' if m.group('type') == 'audio' else None
1598                 }]
1599                 info_dict['direct'] = True
1600             self._sort_formats(formats)
1601             info_dict['formats'] = formats
1602             return info_dict
1603
1604         if not self._downloader.params.get('test', False) and not is_intentional:
1605             force = self._downloader.params.get('force_generic_extractor', False)
1606             self._downloader.report_warning(
1607                 '%s on generic information extractor.' % ('Forcing' if force else 'Falling back'))
1608
1609         if not full_response:
1610             request = sanitized_Request(url)
1611             # Some webservers may serve compressed content of rather big size (e.g. gzipped flac)
1612             # making it impossible to download only chunk of the file (yet we need only 512kB to
1613             # test whether it's HTML or not). According to youtube-dl default Accept-Encoding
1614             # that will always result in downloading the whole file that is not desirable.
1615             # Therefore for extraction pass we have to override Accept-Encoding to any in order
1616             # to accept raw bytes and being able to download only a chunk.
1617             # It may probably better to solve this by checking Content-Type for application/octet-stream
1618             # after HEAD request finishes, but not sure if we can rely on this.
1619             request.add_header('Accept-Encoding', '*')
1620             full_response = self._request_webpage(request, video_id)
1621
1622         first_bytes = full_response.read(512)
1623
1624         # Is it an M3U playlist?
1625         if first_bytes.startswith(b'#EXTM3U'):
1626             info_dict['formats'] = self._extract_m3u8_formats(url, video_id, 'mp4')
1627             self._sort_formats(info_dict['formats'])
1628             return info_dict
1629
1630         # Maybe it's a direct link to a video?
1631         # Be careful not to download the whole thing!
1632         if not is_html(first_bytes):
1633             self._downloader.report_warning(
1634                 'URL could be a direct video link, returning it as such.')
1635             info_dict.update({
1636                 'direct': True,
1637                 'url': url,
1638             })
1639             return info_dict
1640
1641         webpage = self._webpage_read_content(
1642             full_response, url, video_id, prefix=first_bytes)
1643
1644         self.report_extraction(video_id)
1645
1646         # Is it an RSS feed, a SMIL file, an XSPF playlist or a MPD manifest?
1647         try:
1648             doc = compat_etree_fromstring(webpage.encode('utf-8'))
1649             if doc.tag == 'rss':
1650                 return self._extract_rss(url, video_id, doc)
1651             elif re.match(r'^(?:{[^}]+})?smil$', doc.tag):
1652                 smil = self._parse_smil(doc, url, video_id)
1653                 self._sort_formats(smil['formats'])
1654                 return smil
1655             elif doc.tag == '{http://xspf.org/ns/0/}playlist':
1656                 return self.playlist_result(self._parse_xspf(doc, video_id), video_id)
1657             elif re.match(r'(?i)^(?:{[^}]+})?MPD$', doc.tag):
1658                 info_dict['formats'] = self._parse_mpd_formats(
1659                     doc, video_id,
1660                     mpd_base_url=full_response.geturl().rpartition('/')[0],
1661                     mpd_url=url)
1662                 self._sort_formats(info_dict['formats'])
1663                 return info_dict
1664             elif re.match(r'^{http://ns\.adobe\.com/f4m/[12]\.0}manifest$', doc.tag):
1665                 info_dict['formats'] = self._parse_f4m_formats(doc, url, video_id)
1666                 self._sort_formats(info_dict['formats'])
1667                 return info_dict
1668         except compat_xml_parse_error:
1669             pass
1670
1671         # Is it a Camtasia project?
1672         camtasia_res = self._extract_camtasia(url, video_id, webpage)
1673         if camtasia_res is not None:
1674             return camtasia_res
1675
1676         # Sometimes embedded video player is hidden behind percent encoding
1677         # (e.g. https://github.com/rg3/youtube-dl/issues/2448)
1678         # Unescaping the whole page allows to handle those cases in a generic way
1679         webpage = compat_urllib_parse_unquote(webpage)
1680
1681         # it's tempting to parse this further, but you would
1682         # have to take into account all the variations like
1683         #   Video Title - Site Name
1684         #   Site Name | Video Title
1685         #   Video Title - Tagline | Site Name
1686         # and so on and so forth; it's just not practical
1687         video_title = self._og_search_title(
1688             webpage, default=None) or self._html_search_regex(
1689             r'(?s)<title>(.*?)</title>', webpage, 'video title',
1690             default='video')
1691
1692         # Try to detect age limit automatically
1693         age_limit = self._rta_search(webpage)
1694         # And then there are the jokers who advertise that they use RTA,
1695         # but actually don't.
1696         AGE_LIMIT_MARKERS = [
1697             r'Proudly Labeled <a href="http://www.rtalabel.org/" title="Restricted to Adults">RTA</a>',
1698         ]
1699         if any(re.search(marker, webpage) for marker in AGE_LIMIT_MARKERS):
1700             age_limit = 18
1701
1702         # video uploader is domain name
1703         video_uploader = self._search_regex(
1704             r'^(?:https?://)?([^/]*)/.*', url, 'video uploader')
1705
1706         video_description = self._og_search_description(webpage, default=None)
1707         video_thumbnail = self._og_search_thumbnail(webpage, default=None)
1708
1709         # Helper method
1710         def _playlist_from_matches(matches, getter=None, ie=None):
1711             urlrs = orderedSet(
1712                 self.url_result(self._proto_relative_url(getter(m) if getter else m), ie)
1713                 for m in matches)
1714             return self.playlist_result(
1715                 urlrs, playlist_id=video_id, playlist_title=video_title)
1716
1717         # Look for Brightcove Legacy Studio embeds
1718         bc_urls = BrightcoveLegacyIE._extract_brightcove_urls(webpage)
1719         if bc_urls:
1720             self.to_screen('Brightcove video detected.')
1721             entries = [{
1722                 '_type': 'url',
1723                 'url': smuggle_url(bc_url, {'Referer': url}),
1724                 'ie_key': 'BrightcoveLegacy'
1725             } for bc_url in bc_urls]
1726
1727             return {
1728                 '_type': 'playlist',
1729                 'title': video_title,
1730                 'id': video_id,
1731                 'entries': entries,
1732             }
1733
1734         # Look for Brightcove New Studio embeds
1735         bc_urls = BrightcoveNewIE._extract_urls(webpage)
1736         if bc_urls:
1737             return _playlist_from_matches(bc_urls, ie='BrightcoveNew')
1738
1739         # Look for ThePlatform embeds
1740         tp_urls = ThePlatformIE._extract_urls(webpage)
1741         if tp_urls:
1742             return _playlist_from_matches(tp_urls, ie='ThePlatform')
1743
1744         # Look for Vessel embeds
1745         vessel_urls = VesselIE._extract_urls(webpage)
1746         if vessel_urls:
1747             return _playlist_from_matches(vessel_urls, ie=VesselIE.ie_key())
1748
1749         # Look for embedded rtl.nl player
1750         matches = re.findall(
1751             r'<iframe[^>]+?src="((?:https?:)?//(?:www\.)?rtl\.nl/system/videoplayer/[^"]+(?:video_)?embed[^"]+)"',
1752             webpage)
1753         if matches:
1754             return _playlist_from_matches(matches, ie='RtlNl')
1755
1756         vimeo_urls = VimeoIE._extract_urls(url, webpage)
1757         if vimeo_urls:
1758             return _playlist_from_matches(vimeo_urls, ie=VimeoIE.ie_key())
1759
1760         vid_me_embed_url = self._search_regex(
1761             r'src=[\'"](https?://vid\.me/[^\'"]+)[\'"]',
1762             webpage, 'vid.me embed', default=None)
1763         if vid_me_embed_url is not None:
1764             return self.url_result(vid_me_embed_url, 'Vidme')
1765
1766         # Look for embedded YouTube player
1767         matches = re.findall(r'''(?x)
1768             (?:
1769                 <iframe[^>]+?src=|
1770                 data-video-url=|
1771                 <embed[^>]+?src=|
1772                 embedSWF\(?:\s*|
1773                 new\s+SWFObject\(
1774             )
1775             (["\'])
1776                 (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
1777                 (?:embed|v|p)/.+?)
1778             \1''', webpage)
1779         if matches:
1780             return _playlist_from_matches(
1781                 matches, lambda m: unescapeHTML(m[1]))
1782
1783         # Look for lazyYT YouTube embed
1784         matches = re.findall(
1785             r'class="lazyYT" data-youtube-id="([^"]+)"', webpage)
1786         if matches:
1787             return _playlist_from_matches(matches, lambda m: unescapeHTML(m))
1788
1789         # Look for Wordpress "YouTube Video Importer" plugin
1790         matches = re.findall(r'''(?x)<div[^>]+
1791             class=(?P<q1>[\'"])[^\'"]*\byvii_single_video_player\b[^\'"]*(?P=q1)[^>]+
1792             data-video_id=(?P<q2>[\'"])([^\'"]+)(?P=q2)''', webpage)
1793         if matches:
1794             return _playlist_from_matches(matches, lambda m: m[-1])
1795
1796         matches = DailymotionIE._extract_urls(webpage)
1797         if matches:
1798             return _playlist_from_matches(matches)
1799
1800         # Look for embedded Dailymotion playlist player (#3822)
1801         m = re.search(
1802             r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?dailymotion\.[a-z]{2,3}/widget/jukebox\?.+?)\1', webpage)
1803         if m:
1804             playlists = re.findall(
1805                 r'list\[\]=/playlist/([^/]+)/', unescapeHTML(m.group('url')))
1806             if playlists:
1807                 return _playlist_from_matches(
1808                     playlists, lambda p: '//dailymotion.com/playlist/%s' % p)
1809
1810         # Look for embedded Wistia player
1811         match = re.search(
1812             r'<(?:meta[^>]+?content|iframe[^>]+?src)=(["\'])(?P<url>(?:https?:)?//(?:fast\.)?wistia\.net/embed/iframe/.+?)\1', webpage)
1813         if match:
1814             embed_url = self._proto_relative_url(
1815                 unescapeHTML(match.group('url')))
1816             return {
1817                 '_type': 'url_transparent',
1818                 'url': embed_url,
1819                 'ie_key': 'Wistia',
1820                 'uploader': video_uploader,
1821             }
1822
1823         match = re.search(r'(?:id=["\']wistia_|data-wistia-?id=["\']|Wistia\.embed\(["\'])(?P<id>[^"\']+)', webpage)
1824         if match:
1825             return {
1826                 '_type': 'url_transparent',
1827                 'url': 'wistia:%s' % match.group('id'),
1828                 'ie_key': 'Wistia',
1829                 'uploader': video_uploader,
1830             }
1831
1832         match = re.search(
1833             r'''(?sx)
1834                 <script[^>]+src=(["'])(?:https?:)?//fast\.wistia\.com/assets/external/E-v1\.js\1[^>]*>.*?
1835                 <div[^>]+class=(["']).*?\bwistia_async_(?P<id>[a-z0-9]+)\b.*?\2
1836             ''', webpage)
1837         if match:
1838             return self.url_result(self._proto_relative_url(
1839                 'wistia:%s' % match.group('id')), 'Wistia')
1840
1841         # Look for SVT player
1842         svt_url = SVTIE._extract_url(webpage)
1843         if svt_url:
1844             return self.url_result(svt_url, 'SVT')
1845
1846         # Look for embedded condenast player
1847         matches = re.findall(
1848             r'<iframe\s+(?:[a-zA-Z-]+="[^"]+"\s+)*?src="(https?://player\.cnevids\.com/embed/[^"]+")',
1849             webpage)
1850         if matches:
1851             return {
1852                 '_type': 'playlist',
1853                 'entries': [{
1854                     '_type': 'url',
1855                     'ie_key': 'CondeNast',
1856                     'url': ma,
1857                 } for ma in matches],
1858                 'title': video_title,
1859                 'id': video_id,
1860             }
1861
1862         # Look for Bandcamp pages with custom domain
1863         mobj = re.search(r'<meta property="og:url"[^>]*?content="(.*?bandcamp\.com.*?)"', webpage)
1864         if mobj is not None:
1865             burl = unescapeHTML(mobj.group(1))
1866             # Don't set the extractor because it can be a track url or an album
1867             return self.url_result(burl)
1868
1869         # Look for embedded Vevo player
1870         mobj = re.search(
1871             r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:cache\.)?vevo\.com/.+?)\1', webpage)
1872         if mobj is not None:
1873             return self.url_result(mobj.group('url'))
1874
1875         # Look for embedded Viddler player
1876         mobj = re.search(
1877             r'<(?:iframe[^>]+?src|param[^>]+?value)=(["\'])(?P<url>(?:https?:)?//(?:www\.)?viddler\.com/(?:embed|player)/.+?)\1',
1878             webpage)
1879         if mobj is not None:
1880             return self.url_result(mobj.group('url'))
1881
1882         # Look for NYTimes player
1883         mobj = re.search(
1884             r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//graphics8\.nytimes\.com/bcvideo/[^/]+/iframe/embed\.html.+?)\1>',
1885             webpage)
1886         if mobj is not None:
1887             return self.url_result(mobj.group('url'))
1888
1889         # Look for Libsyn player
1890         mobj = re.search(
1891             r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//html5-player\.libsyn\.com/embed/.+?)\1', webpage)
1892         if mobj is not None:
1893             return self.url_result(mobj.group('url'))
1894
1895         # Look for Ooyala videos
1896         mobj = (re.search(r'player\.ooyala\.com/[^"?]+[?#][^"]*?(?:embedCode|ec)=(?P<ec>[^"&]+)', webpage) or
1897                 re.search(r'OO\.Player\.create\([\'"].*?[\'"],\s*[\'"](?P<ec>.{32})[\'"]', webpage) or
1898                 re.search(r'SBN\.VideoLinkset\.ooyala\([\'"](?P<ec>.{32})[\'"]\)', webpage) or
1899                 re.search(r'data-ooyala-video-id\s*=\s*[\'"](?P<ec>.{32})[\'"]', webpage))
1900         if mobj is not None:
1901             return OoyalaIE._build_url_result(smuggle_url(mobj.group('ec'), {'domain': url}))
1902
1903         # Look for multiple Ooyala embeds on SBN network websites
1904         mobj = re.search(r'SBN\.VideoLinkset\.entryGroup\((\[.*?\])', webpage)
1905         if mobj is not None:
1906             embeds = self._parse_json(mobj.group(1), video_id, fatal=False)
1907             if embeds:
1908                 return _playlist_from_matches(
1909                     embeds, getter=lambda v: OoyalaIE._url_for_embed_code(smuggle_url(v['provider_video_id'], {'domain': url})), ie='Ooyala')
1910
1911         # Look for Aparat videos
1912         mobj = re.search(r'<iframe .*?src="(http://www\.aparat\.com/video/[^"]+)"', webpage)
1913         if mobj is not None:
1914             return self.url_result(mobj.group(1), 'Aparat')
1915
1916         # Look for MPORA videos
1917         mobj = re.search(r'<iframe .*?src="(http://mpora\.(?:com|de)/videos/[^"]+)"', webpage)
1918         if mobj is not None:
1919             return self.url_result(mobj.group(1), 'Mpora')
1920
1921         # Look for embedded NovaMov-based player
1922         mobj = re.search(
1923             r'''(?x)<(?:pagespeed_)?iframe[^>]+?src=(["\'])
1924                     (?P<url>http://(?:(?:embed|www)\.)?
1925                         (?:novamov\.com|
1926                            nowvideo\.(?:ch|sx|eu|at|ag|co)|
1927                            videoweed\.(?:es|com)|
1928                            movshare\.(?:net|sx|ag)|
1929                            divxstage\.(?:eu|net|ch|co|at|ag))
1930                         /embed\.php.+?)\1''', webpage)
1931         if mobj is not None:
1932             return self.url_result(mobj.group('url'))
1933
1934         # Look for embedded Facebook player
1935         facebook_url = FacebookIE._extract_url(webpage)
1936         if facebook_url is not None:
1937             return self.url_result(facebook_url, 'Facebook')
1938
1939         # Look for embedded VK player
1940         mobj = re.search(r'<iframe[^>]+?src=(["\'])(?P<url>https?://vk\.com/video_ext\.php.+?)\1', webpage)
1941         if mobj is not None:
1942             return self.url_result(mobj.group('url'), 'VK')
1943
1944         # Look for embedded Odnoklassniki player
1945         mobj = re.search(r'<iframe[^>]+?src=(["\'])(?P<url>https?://(?:odnoklassniki|ok)\.ru/videoembed/.+?)\1', webpage)
1946         if mobj is not None:
1947             return self.url_result(mobj.group('url'), 'Odnoklassniki')
1948
1949         # Look for embedded ivi player
1950         mobj = re.search(r'<embed[^>]+?src=(["\'])(?P<url>https?://(?:www\.)?ivi\.ru/video/player.+?)\1', webpage)
1951         if mobj is not None:
1952             return self.url_result(mobj.group('url'), 'Ivi')
1953
1954         # Look for embedded Huffington Post player
1955         mobj = re.search(
1956             r'<iframe[^>]+?src=(["\'])(?P<url>https?://embed\.live\.huffingtonpost\.com/.+?)\1', webpage)
1957         if mobj is not None:
1958             return self.url_result(mobj.group('url'), 'HuffPost')
1959
1960         # Look for embed.ly
1961         mobj = re.search(r'class=["\']embedly-card["\'][^>]href=["\'](?P<url>[^"\']+)', webpage)
1962         if mobj is not None:
1963             return self.url_result(mobj.group('url'))
1964         mobj = re.search(r'class=["\']embedly-embed["\'][^>]src=["\'][^"\']*url=(?P<url>[^&]+)', webpage)
1965         if mobj is not None:
1966             return self.url_result(compat_urllib_parse_unquote(mobj.group('url')))
1967
1968         # Look for funnyordie embed
1969         matches = re.findall(r'<iframe[^>]+?src="(https?://(?:www\.)?funnyordie\.com/embed/[^"]+)"', webpage)
1970         if matches:
1971             return _playlist_from_matches(
1972                 matches, getter=unescapeHTML, ie='FunnyOrDie')
1973
1974         # Look for BBC iPlayer embed
1975         matches = re.findall(r'setPlaylist\("(https?://www\.bbc\.co\.uk/iplayer/[^/]+/[\da-z]{8})"\)', webpage)
1976         if matches:
1977             return _playlist_from_matches(matches, ie='BBCCoUk')
1978
1979         # Look for embedded RUTV player
1980         rutv_url = RUTVIE._extract_url(webpage)
1981         if rutv_url:
1982             return self.url_result(rutv_url, 'RUTV')
1983
1984         # Look for embedded TVC player
1985         tvc_url = TVCIE._extract_url(webpage)
1986         if tvc_url:
1987             return self.url_result(tvc_url, 'TVC')
1988
1989         # Look for embedded SportBox player
1990         sportbox_urls = SportBoxEmbedIE._extract_urls(webpage)
1991         if sportbox_urls:
1992             return _playlist_from_matches(sportbox_urls, ie='SportBoxEmbed')
1993
1994         # Look for embedded PornHub player
1995         pornhub_url = PornHubIE._extract_url(webpage)
1996         if pornhub_url:
1997             return self.url_result(pornhub_url, 'PornHub')
1998
1999         # Look for embedded XHamster player
2000         xhamster_urls = XHamsterEmbedIE._extract_urls(webpage)
2001         if xhamster_urls:
2002             return _playlist_from_matches(xhamster_urls, ie='XHamsterEmbed')
2003
2004         # Look for embedded TNAFlixNetwork player
2005         tnaflix_urls = TNAFlixNetworkEmbedIE._extract_urls(webpage)
2006         if tnaflix_urls:
2007             return _playlist_from_matches(tnaflix_urls, ie=TNAFlixNetworkEmbedIE.ie_key())
2008
2009         # Look for embedded Tvigle player
2010         mobj = re.search(
2011             r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//cloud\.tvigle\.ru/video/.+?)\1', webpage)
2012         if mobj is not None:
2013             return self.url_result(mobj.group('url'), 'Tvigle')
2014
2015         # Look for embedded TED player
2016         mobj = re.search(
2017             r'<iframe[^>]+?src=(["\'])(?P<url>https?://embed(?:-ssl)?\.ted\.com/.+?)\1', webpage)
2018         if mobj is not None:
2019             return self.url_result(mobj.group('url'), 'TED')
2020
2021         # Look for embedded Ustream videos
2022         mobj = re.search(
2023             r'<iframe[^>]+?src=(["\'])(?P<url>http://www\.ustream\.tv/embed/.+?)\1', webpage)
2024         if mobj is not None:
2025             return self.url_result(mobj.group('url'), 'Ustream')
2026
2027         # Look for embedded arte.tv player
2028         mobj = re.search(
2029             r'<(?:script|iframe) [^>]*?src="(?P<url>http://www\.arte\.tv/(?:playerv2/embed|arte_vp/index)[^"]+)"',
2030             webpage)
2031         if mobj is not None:
2032             return self.url_result(mobj.group('url'), 'ArteTVEmbed')
2033
2034         # Look for embedded francetv player
2035         mobj = re.search(
2036             r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?://)?embed\.francetv\.fr/\?ue=.+?)\1',
2037             webpage)
2038         if mobj is not None:
2039             return self.url_result(mobj.group('url'))
2040
2041         # Look for embedded smotri.com player
2042         smotri_url = SmotriIE._extract_url(webpage)
2043         if smotri_url:
2044             return self.url_result(smotri_url, 'Smotri')
2045
2046         # Look for embedded Myvi.ru player
2047         myvi_url = MyviIE._extract_url(webpage)
2048         if myvi_url:
2049             return self.url_result(myvi_url)
2050
2051         # Look for embedded soundcloud player
2052         soundcloud_urls = SoundcloudIE._extract_urls(webpage)
2053         if soundcloud_urls:
2054             return _playlist_from_matches(soundcloud_urls, getter=unescapeHTML, ie=SoundcloudIE.ie_key())
2055
2056         # Look for embedded mtvservices player
2057         mtvservices_url = MTVServicesEmbeddedIE._extract_url(webpage)
2058         if mtvservices_url:
2059             return self.url_result(mtvservices_url, ie='MTVServicesEmbedded')
2060
2061         # Look for embedded yahoo player
2062         mobj = re.search(
2063             r'<iframe[^>]+?src=(["\'])(?P<url>https?://(?:screen|movies)\.yahoo\.com/.+?\.html\?format=embed)\1',
2064             webpage)
2065         if mobj is not None:
2066             return self.url_result(mobj.group('url'), 'Yahoo')
2067
2068         # Look for embedded sbs.com.au player
2069         mobj = re.search(
2070             r'''(?x)
2071             (?:
2072                 <meta\s+property="og:video"\s+content=|
2073                 <iframe[^>]+?src=
2074             )
2075             (["\'])(?P<url>https?://(?:www\.)?sbs\.com\.au/ondemand/video/.+?)\1''',
2076             webpage)
2077         if mobj is not None:
2078             return self.url_result(mobj.group('url'), 'SBS')
2079
2080         # Look for embedded Cinchcast player
2081         mobj = re.search(
2082             r'<iframe[^>]+?src=(["\'])(?P<url>https?://player\.cinchcast\.com/.+?)\1',
2083             webpage)
2084         if mobj is not None:
2085             return self.url_result(mobj.group('url'), 'Cinchcast')
2086
2087         mobj = re.search(
2088             r'<iframe[^>]+?src=(["\'])(?P<url>https?://m(?:lb)?\.mlb\.com/shared/video/embed/embed\.html\?.+?)\1',
2089             webpage)
2090         if not mobj:
2091             mobj = re.search(
2092                 r'data-video-link=["\'](?P<url>http://m.mlb.com/video/[^"\']+)',
2093                 webpage)
2094         if mobj is not None:
2095             return self.url_result(mobj.group('url'), 'MLB')
2096
2097         mobj = re.search(
2098             r'<(?:iframe|script)[^>]+?src=(["\'])(?P<url>%s)\1' % CondeNastIE.EMBED_URL,
2099             webpage)
2100         if mobj is not None:
2101             return self.url_result(self._proto_relative_url(mobj.group('url'), scheme='http:'), 'CondeNast')
2102
2103         mobj = re.search(
2104             r'<iframe[^>]+src="(?P<url>https?://(?:new\.)?livestream\.com/[^"]+/player[^"]+)"',
2105             webpage)
2106         if mobj is not None:
2107             return self.url_result(mobj.group('url'), 'Livestream')
2108
2109         # Look for Zapiks embed
2110         mobj = re.search(
2111             r'<iframe[^>]+src="(?P<url>https?://(?:www\.)?zapiks\.fr/index\.php\?.+?)"', webpage)
2112         if mobj is not None:
2113             return self.url_result(mobj.group('url'), 'Zapiks')
2114
2115         # Look for Kaltura embeds
2116         kaltura_url = KalturaIE._extract_url(webpage)
2117         if kaltura_url:
2118             return self.url_result(smuggle_url(kaltura_url, {'source_url': url}), KalturaIE.ie_key())
2119
2120         # Look for Eagle.Platform embeds
2121         eagleplatform_url = EaglePlatformIE._extract_url(webpage)
2122         if eagleplatform_url:
2123             return self.url_result(eagleplatform_url, EaglePlatformIE.ie_key())
2124
2125         # Look for ClipYou (uses Eagle.Platform) embeds
2126         mobj = re.search(
2127             r'<iframe[^>]+src="https?://(?P<host>media\.clipyou\.ru)/index/player\?.*\brecord_id=(?P<id>\d+).*"', webpage)
2128         if mobj is not None:
2129             return self.url_result('eagleplatform:%(host)s:%(id)s' % mobj.groupdict(), 'EaglePlatform')
2130
2131         # Look for Pladform embeds
2132         pladform_url = PladformIE._extract_url(webpage)
2133         if pladform_url:
2134             return self.url_result(pladform_url)
2135
2136         # Look for Videomore embeds
2137         videomore_url = VideomoreIE._extract_url(webpage)
2138         if videomore_url:
2139             return self.url_result(videomore_url)
2140
2141         # Look for Playwire embeds
2142         mobj = re.search(
2143             r'<script[^>]+data-config=(["\'])(?P<url>(?:https?:)?//config\.playwire\.com/.+?)\1', webpage)
2144         if mobj is not None:
2145             return self.url_result(mobj.group('url'))
2146
2147         # Look for 5min embeds
2148         mobj = re.search(
2149             r'<meta[^>]+property="og:video"[^>]+content="https?://embed\.5min\.com/(?P<id>[0-9]+)/?', webpage)
2150         if mobj is not None:
2151             return self.url_result('5min:%s' % mobj.group('id'), 'FiveMin')
2152
2153         # Look for Crooks and Liars embeds
2154         mobj = re.search(
2155             r'<(?:iframe[^>]+src|param[^>]+value)=(["\'])(?P<url>(?:https?:)?//embed\.crooksandliars\.com/(?:embed|v)/.+?)\1', webpage)
2156         if mobj is not None:
2157             return self.url_result(mobj.group('url'))
2158
2159         # Look for NBC Sports VPlayer embeds
2160         nbc_sports_url = NBCSportsVPlayerIE._extract_url(webpage)
2161         if nbc_sports_url:
2162             return self.url_result(nbc_sports_url, 'NBCSportsVPlayer')
2163
2164         # Look for NBC News embeds
2165         nbc_news_embed_url = re.search(
2166             r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//www\.nbcnews\.com/widget/video-embed/[^"\']+)\1', webpage)
2167         if nbc_news_embed_url:
2168             return self.url_result(nbc_news_embed_url.group('url'), 'NBCNews')
2169
2170         # Look for Google Drive embeds
2171         google_drive_url = GoogleDriveIE._extract_url(webpage)
2172         if google_drive_url:
2173             return self.url_result(google_drive_url, 'GoogleDrive')
2174
2175         # Look for UDN embeds
2176         mobj = re.search(
2177             r'<iframe[^>]+src="(?P<url>%s)"' % UDNEmbedIE._PROTOCOL_RELATIVE_VALID_URL, webpage)
2178         if mobj is not None:
2179             return self.url_result(
2180                 compat_urlparse.urljoin(url, mobj.group('url')), 'UDNEmbed')
2181
2182         # Look for Senate ISVP iframe
2183         senate_isvp_url = SenateISVPIE._search_iframe_url(webpage)
2184         if senate_isvp_url:
2185             return self.url_result(senate_isvp_url, 'SenateISVP')
2186
2187         # Look for Dailymotion Cloud videos
2188         dmcloud_url = DailymotionCloudIE._extract_dmcloud_url(webpage)
2189         if dmcloud_url:
2190             return self.url_result(dmcloud_url, 'DailymotionCloud')
2191
2192         # Look for OnionStudios embeds
2193         onionstudios_url = OnionStudiosIE._extract_url(webpage)
2194         if onionstudios_url:
2195             return self.url_result(onionstudios_url)
2196
2197         # Look for ViewLift embeds
2198         viewlift_url = ViewLiftEmbedIE._extract_url(webpage)
2199         if viewlift_url:
2200             return self.url_result(viewlift_url)
2201
2202         # Look for JWPlatform embeds
2203         jwplatform_url = JWPlatformIE._extract_url(webpage)
2204         if jwplatform_url:
2205             return self.url_result(jwplatform_url, 'JWPlatform')
2206
2207         # Look for ScreenwaveMedia embeds
2208         mobj = re.search(ScreenwaveMediaIE.EMBED_PATTERN, webpage)
2209         if mobj is not None:
2210             return self.url_result(unescapeHTML(mobj.group('url')), 'ScreenwaveMedia')
2211
2212         # Look for Digiteka embeds
2213         digiteka_url = DigitekaIE._extract_url(webpage)
2214         if digiteka_url:
2215             return self.url_result(self._proto_relative_url(digiteka_url), DigitekaIE.ie_key())
2216
2217         # Look for Arkena embeds
2218         arkena_url = ArkenaIE._extract_url(webpage)
2219         if arkena_url:
2220             return self.url_result(arkena_url, ArkenaIE.ie_key())
2221
2222         # Look for Limelight embeds
2223         mobj = re.search(r'LimelightPlayer\.doLoad(Media|Channel|ChannelList)\(["\'](?P<id>[a-z0-9]{32})', webpage)
2224         if mobj:
2225             lm = {
2226                 'Media': 'media',
2227                 'Channel': 'channel',
2228                 'ChannelList': 'channel_list',
2229             }
2230             return self.url_result('limelight:%s:%s' % (
2231                 lm[mobj.group(1)], mobj.group(2)), 'Limelight%s' % mobj.group(1), mobj.group(2))
2232
2233         # Look for AdobeTVVideo embeds
2234         mobj = re.search(
2235             r'<iframe[^>]+src=[\'"]((?:https?:)?//video\.tv\.adobe\.com/v/\d+[^"]+)[\'"]',
2236             webpage)
2237         if mobj is not None:
2238             return self.url_result(
2239                 self._proto_relative_url(unescapeHTML(mobj.group(1))),
2240                 'AdobeTVVideo')
2241
2242         # Look for Vine embeds
2243         mobj = re.search(
2244             r'<iframe[^>]+src=[\'"]((?:https?:)?//(?:www\.)?vine\.co/v/[^/]+/embed/(?:simple|postcard))',
2245             webpage)
2246         if mobj is not None:
2247             return self.url_result(
2248                 self._proto_relative_url(unescapeHTML(mobj.group(1))), 'Vine')
2249
2250         # Look for VODPlatform embeds
2251         mobj = re.search(
2252             r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?vod-platform\.net/[eE]mbed/.+?)\1',
2253             webpage)
2254         if mobj is not None:
2255             return self.url_result(
2256                 self._proto_relative_url(unescapeHTML(mobj.group('url'))), 'VODPlatform')
2257
2258         # Look for Mangomolo embeds
2259         mobj = re.search(
2260             r'''(?x)<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?admin\.mangomolo\.com/analytics/index\.php/customers/embed/
2261                 (?:
2262                     video\?.*?\bid=(?P<video_id>\d+)|
2263                     index\?.*?\bchannelid=(?P<channel_id>(?:[A-Za-z0-9+/=]|%2B|%2F|%3D)+)
2264                 ).+?)\1''', webpage)
2265         if mobj is not None:
2266             info = {
2267                 '_type': 'url_transparent',
2268                 'url': self._proto_relative_url(unescapeHTML(mobj.group('url'))),
2269                 'title': video_title,
2270                 'description': video_description,
2271                 'thumbnail': video_thumbnail,
2272                 'uploader': video_uploader,
2273             }
2274             video_id = mobj.group('video_id')
2275             if video_id:
2276                 info.update({
2277                     'ie_key': 'MangomoloVideo',
2278                     'id': video_id,
2279                 })
2280             else:
2281                 info.update({
2282                     'ie_key': 'MangomoloLive',
2283                     'id': mobj.group('channel_id'),
2284                 })
2285             return info
2286
2287         # Look for Instagram embeds
2288         instagram_embed_url = InstagramIE._extract_embed_url(webpage)
2289         if instagram_embed_url is not None:
2290             return self.url_result(
2291                 self._proto_relative_url(instagram_embed_url), InstagramIE.ie_key())
2292
2293         # Look for LiveLeak embeds
2294         liveleak_url = LiveLeakIE._extract_url(webpage)
2295         if liveleak_url:
2296             return self.url_result(liveleak_url, 'LiveLeak')
2297
2298         # Look for 3Q SDN embeds
2299         threeqsdn_url = ThreeQSDNIE._extract_url(webpage)
2300         if threeqsdn_url:
2301             return {
2302                 '_type': 'url_transparent',
2303                 'ie_key': ThreeQSDNIE.ie_key(),
2304                 'url': self._proto_relative_url(threeqsdn_url),
2305                 'title': video_title,
2306                 'description': video_description,
2307                 'thumbnail': video_thumbnail,
2308                 'uploader': video_uploader,
2309             }
2310
2311         # Look for VBOX7 embeds
2312         vbox7_url = Vbox7IE._extract_url(webpage)
2313         if vbox7_url:
2314             return self.url_result(vbox7_url, Vbox7IE.ie_key())
2315
2316         # Look for DBTV embeds
2317         dbtv_urls = DBTVIE._extract_urls(webpage)
2318         if dbtv_urls:
2319             return _playlist_from_matches(dbtv_urls, ie=DBTVIE.ie_key())
2320
2321         # Looking for http://schema.org/VideoObject
2322         json_ld = self._search_json_ld(
2323             webpage, video_id, default={}, expected_type='VideoObject')
2324         if json_ld.get('url'):
2325             info_dict.update({
2326                 'title': video_title or info_dict['title'],
2327                 'description': video_description,
2328                 'thumbnail': video_thumbnail,
2329                 'age_limit': age_limit
2330             })
2331             info_dict.update(json_ld)
2332             return info_dict
2333
2334         # Look for HTML5 media
2335         entries = self._parse_html5_media_entries(url, webpage, video_id, m3u8_id='hls')
2336         if entries:
2337             for entry in entries:
2338                 entry.update({
2339                     'id': video_id,
2340                     'title': video_title,
2341                 })
2342                 self._sort_formats(entry['formats'])
2343             return self.playlist_result(entries)
2344
2345         def check_video(vurl):
2346             if YoutubeIE.suitable(vurl):
2347                 return True
2348             vpath = compat_urlparse.urlparse(vurl).path
2349             vext = determine_ext(vpath)
2350             return '.' in vpath and vext not in ('swf', 'png', 'jpg', 'srt', 'sbv', 'sub', 'vtt', 'ttml', 'js')
2351
2352         def filter_video(urls):
2353             return list(filter(check_video, urls))
2354
2355         # Start with something easy: JW Player in SWFObject
2356         found = filter_video(re.findall(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage))
2357         if not found:
2358             # Look for gorilla-vid style embedding
2359             found = filter_video(re.findall(r'''(?sx)
2360                 (?:
2361                     jw_plugins|
2362                     JWPlayerOptions|
2363                     jwplayer\s*\(\s*["'][^'"]+["']\s*\)\s*\.setup
2364                 )
2365                 .*?
2366                 ['"]?file['"]?\s*:\s*["\'](.*?)["\']''', webpage))
2367         if not found:
2368             # Broaden the search a little bit
2369             found = filter_video(re.findall(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage))
2370         if not found:
2371             # Broaden the findall a little bit: JWPlayer JS loader
2372             found = filter_video(re.findall(
2373                 r'[^A-Za-z0-9]?(?:file|video_url)["\']?:\s*["\'](http(?![^\'"]+\.[0-9]+[\'"])[^\'"]+)["\']', webpage))
2374         if not found:
2375             # Flow player
2376             found = filter_video(re.findall(r'''(?xs)
2377                 flowplayer\("[^"]+",\s*
2378                     \{[^}]+?\}\s*,
2379                     \s*\{[^}]+? ["']?clip["']?\s*:\s*\{\s*
2380                         ["']?url["']?\s*:\s*["']([^"']+)["']
2381             ''', webpage))
2382         if not found:
2383             # Cinerama player
2384             found = re.findall(
2385                 r"cinerama\.embedPlayer\(\s*\'[^']+\',\s*'([^']+)'", webpage)
2386         if not found:
2387             # Try to find twitter cards info
2388             # twitter:player:stream should be checked before twitter:player since
2389             # it is expected to contain a raw stream (see
2390             # https://dev.twitter.com/cards/types/player#On_twitter.com_via_desktop_browser)
2391             found = filter_video(re.findall(
2392                 r'<meta (?:property|name)="twitter:player:stream" (?:content|value)="(.+?)"', webpage))
2393         if not found:
2394             # We look for Open Graph info:
2395             # We have to match any number spaces between elements, some sites try to align them (eg.: statigr.am)
2396             m_video_type = re.findall(r'<meta.*?property="og:video:type".*?content="video/(.*?)"', webpage)
2397             # We only look in og:video if the MIME type is a video, don't try if it's a Flash player:
2398             if m_video_type is not None:
2399                 found = filter_video(re.findall(r'<meta.*?property="og:video".*?content="(.*?)"', webpage))
2400         if not found:
2401             REDIRECT_REGEX = r'[0-9]{,2};\s*(?:URL|url)=\'?([^\'"]+)'
2402             found = re.search(
2403                 r'(?i)<meta\s+(?=(?:[a-z-]+="[^"]+"\s+)*http-equiv="refresh")'
2404                 r'(?:[a-z-]+="[^"]+"\s+)*?content="%s' % REDIRECT_REGEX,
2405                 webpage)
2406             if not found:
2407                 # Look also in Refresh HTTP header
2408                 refresh_header = head_response.headers.get('Refresh')
2409                 if refresh_header:
2410                     # In python 2 response HTTP headers are bytestrings
2411                     if sys.version_info < (3, 0) and isinstance(refresh_header, str):
2412                         refresh_header = refresh_header.decode('iso-8859-1')
2413                     found = re.search(REDIRECT_REGEX, refresh_header)
2414             if found:
2415                 new_url = compat_urlparse.urljoin(url, unescapeHTML(found.group(1)))
2416                 self.report_following_redirect(new_url)
2417                 return {
2418                     '_type': 'url',
2419                     'url': new_url,
2420                 }
2421
2422         if not found:
2423             # twitter:player is a https URL to iframe player that may or may not
2424             # be supported by youtube-dl thus this is checked the very last (see
2425             # https://dev.twitter.com/cards/types/player#On_twitter.com_via_desktop_browser)
2426             embed_url = self._html_search_meta('twitter:player', webpage, default=None)
2427             if embed_url:
2428                 return self.url_result(embed_url)
2429
2430         if not found:
2431             raise UnsupportedError(url)
2432
2433         entries = []
2434         for video_url in orderedSet(found):
2435             video_url = unescapeHTML(video_url)
2436             video_url = video_url.replace('\\/', '/')
2437             video_url = compat_urlparse.urljoin(url, video_url)
2438             video_id = compat_urllib_parse_unquote(os.path.basename(video_url))
2439
2440             # Sometimes, jwplayer extraction will result in a YouTube URL
2441             if YoutubeIE.suitable(video_url):
2442                 entries.append(self.url_result(video_url, 'Youtube'))
2443                 continue
2444
2445             # here's a fun little line of code for you:
2446             video_id = os.path.splitext(video_id)[0]
2447
2448             entry_info_dict = {
2449                 'id': video_id,
2450                 'uploader': video_uploader,
2451                 'title': video_title,
2452                 'age_limit': age_limit,
2453             }
2454
2455             ext = determine_ext(video_url)
2456             if ext == 'smil':
2457                 entry_info_dict['formats'] = self._extract_smil_formats(video_url, video_id)
2458             elif ext == 'xspf':
2459                 return self.playlist_result(self._extract_xspf_playlist(video_url, video_id), video_id)
2460             elif ext == 'm3u8':
2461                 entry_info_dict['formats'] = self._extract_m3u8_formats(video_url, video_id, ext='mp4')
2462             elif ext == 'mpd':
2463                 entry_info_dict['formats'] = self._extract_mpd_formats(video_url, video_id)
2464             elif ext == 'f4m':
2465                 entry_info_dict['formats'] = self._extract_f4m_formats(video_url, video_id)
2466             else:
2467                 entry_info_dict['url'] = video_url
2468
2469             if entry_info_dict.get('formats'):
2470                 self._sort_formats(entry_info_dict['formats'])
2471
2472             entries.append(entry_info_dict)
2473
2474         if len(entries) == 1:
2475             return entries[0]
2476         else:
2477             for num, e in enumerate(entries, start=1):
2478                 # 'url' results don't have a title
2479                 if e.get('title') is not None:
2480                     e['title'] = '%s (%d)' % (e['title'], num)
2481             return {
2482                 '_type': 'playlist',
2483                 'entries': entries,
2484             }