3 from __future__
import unicode_literals
9 from .common
import InfoExtractor
10 from .youtube
import YoutubeIE
11 from ..compat
import (
12 compat_etree_fromstring
,
13 compat_urllib_parse_unquote
,
15 compat_xml_parse_error
,
32 from .brightcove
import (
36 from .nbc
import NBCSportsVPlayerIE
37 from .ooyala
import OoyalaIE
38 from .rutv
import RUTVIE
39 from .tvc
import TVCIE
40 from .sportbox
import SportBoxEmbedIE
41 from .smotri
import SmotriIE
42 from .myvi
import MyviIE
43 from .condenast
import CondeNastIE
44 from .udn
import UDNEmbedIE
45 from .senateisvp
import SenateISVPIE
46 from .svt
import SVTIE
47 from .pornhub
import PornHubIE
48 from .xhamster
import XHamsterEmbedIE
49 from .tnaflix
import TNAFlixNetworkEmbedIE
50 from .drtuber
import DrTuberIE
51 from .redtube
import RedTubeIE
52 from .vimeo
import VimeoIE
53 from .dailymotion
import (
57 from .onionstudios
import OnionStudiosIE
58 from .viewlift
import ViewLiftEmbedIE
59 from .mtv
import MTVServicesEmbeddedIE
60 from .pladform
import PladformIE
61 from .videomore
import VideomoreIE
62 from .webcaster
import WebcasterFeedIE
63 from .googledrive
import GoogleDriveIE
64 from .jwplatform
import JWPlatformIE
65 from .digiteka
import DigitekaIE
66 from .arkena
import ArkenaIE
67 from .instagram
import InstagramIE
68 from .liveleak
import LiveLeakIE
69 from .threeqsdn
import ThreeQSDNIE
70 from .theplatform
import ThePlatformIE
71 from .vessel
import VesselIE
72 from .kaltura
import KalturaIE
73 from .eagleplatform
import EaglePlatformIE
74 from .facebook
import FacebookIE
75 from .soundcloud
import SoundcloudIE
76 from .vbox7
import Vbox7IE
77 from .dbtv
import DBTVIE
78 from .piksel
import PikselIE
81 class GenericIE(InfoExtractor
):
82 IE_DESC
= 'Generic downloader that works on some sites'
86 # Direct link to a video
88 'url': 'http://media.w3.org/2010/05/sintel/trailer.mp4',
89 'md5': '67d406c2bcb6af27fa886f31aa934bbe',
94 'upload_date': '20100513',
97 # Direct link to media delivered compressed (until Accept-Encoding is *)
99 'url': 'http://calimero.tk/muzik/FictionJunction-Parallel_Hearts.flac',
100 'md5': '128c42e68b13950268b648275386fc74',
102 'id': 'FictionJunction-Parallel_Hearts',
104 'title': 'FictionJunction-Parallel_Hearts',
105 'upload_date': '20140522',
107 'expected_warnings': [
108 'URL could be a direct video link, returning it as such.'
110 'skip': 'URL invalid',
112 # Direct download with broken HEAD
114 'url': 'http://ai-radio.org:8000/radio.opus',
121 'skip_download': True, # infinite live stream
123 'expected_warnings': [
124 r
'501.*Not Implemented',
128 # Direct link with incorrect MIME type
130 'url': 'http://ftp.nluug.nl/video/nluug/2014-11-20_nj14/zaal-2/5_Lennart_Poettering_-_Systemd.webm',
131 'md5': '4ccbebe5f36706d85221f204d7eb5913',
133 'url': 'http://ftp.nluug.nl/video/nluug/2014-11-20_nj14/zaal-2/5_Lennart_Poettering_-_Systemd.webm',
134 'id': '5_Lennart_Poettering_-_Systemd',
136 'title': '5_Lennart_Poettering_-_Systemd',
137 'upload_date': '20141120',
139 'expected_warnings': [
140 'URL could be a direct video link, returning it as such.'
145 'url': 'http://phihag.de/2014/youtube-dl/rss2.xml',
147 'id': 'http://phihag.de/2014/youtube-dl/rss2.xml',
148 'title': 'Zero Punctuation',
149 'description': 're:.*groundbreaking video review series.*'
151 'playlist_mincount': 11,
153 # RSS feed with enclosure
155 'url': 'http://podcastfeeds.nbcnews.com/audio/podcast/MSNBC-MADDOW-NETCAST-M4V.xml',
157 'id': 'pdv_maddow_netcast_m4v-02-27-2015-201624',
159 'upload_date': '20150228',
160 'title': 'pdv_maddow_netcast_m4v-02-27-2015-201624',
163 # SMIL from http://videolectures.net/promogram_igor_mekjavic_eng
165 'url': 'http://videolectures.net/promogram_igor_mekjavic_eng/video/1/smil.xml',
169 'title': 'Automatics, robotics and biocybernetics',
170 'description': 'md5:815fc1deb6b3a2bff99de2d5325be482',
171 'upload_date': '20130627',
172 'formats': 'mincount:16',
173 'subtitles': 'mincount:1',
176 'force_generic_extractor': True,
177 'skip_download': True,
180 # SMIL from http://www1.wdr.de/mediathek/video/livestream/index.html
182 'url': 'http://metafilegenerator.de/WDR/WDR_FS/hds/hds.smil',
187 'formats': 'mincount:1',
190 'skip_download': True,
193 # SMIL from https://www.restudy.dk/video/play/id/1637
195 'url': 'https://www.restudy.dk/awsmedia/SmilDirectory/video_1637.xml',
199 'title': 'video_1637',
200 'formats': 'mincount:3',
203 'skip_download': True,
206 # SMIL from http://adventure.howstuffworks.com/5266-cool-jobs-iditarod-musher-video.htm
208 'url': 'http://services.media.howstuffworks.com/videos/450221/smil-service.smil',
210 'id': 'smil-service',
212 'title': 'smil-service',
213 'formats': 'mincount:1',
216 'skip_download': True,
219 # SMIL from http://new.livestream.com/CoheedandCambria/WebsterHall/videos/4719370
221 'url': 'http://api.new.livestream.com/accounts/1570303/events/1585861/videos/4719370.smil',
225 'title': '571de1fd-47bc-48db-abf9-238872a58d1f',
226 'formats': 'mincount:3',
229 'skip_download': True,
232 # XSPF playlist from http://www.telegraaf.nl/tv/nieuws/binnenland/24353229/__Tikibad_ontruimd_wegens_brand__.html
234 'url': 'http://www.telegraaf.nl/xml/playlist/2015/8/7/mZlp2ctYIUEB.xspf',
236 'id': 'mZlp2ctYIUEB',
238 'title': 'Tikibad ontruimd wegens brand',
239 'description': 'md5:05ca046ff47b931f9b04855015e163a4',
240 'thumbnail': 're:^https?://.*\.jpg$',
244 'skip_download': True,
247 # MPD from http://dash-mse-test.appspot.com/media.html
249 'url': 'http://yt-dash-mse-test.commondatastorage.googleapis.com/media/car-20120827-manifest.mpd',
250 'md5': '4b57baab2e30d6eb3a6a09f0ba57ef53',
252 'id': 'car-20120827-manifest',
254 'title': 'car-20120827-manifest',
255 'formats': 'mincount:9',
256 'upload_date': '20130904',
259 'format': 'bestvideo',
262 # m3u8 served with Content-Type: audio/x-mpegURL; charset=utf-8
264 'url': 'http://once.unicornmedia.com/now/master/playlist/bb0b18ba-64f5-4b1b-a29f-0ac252f06b68/77a785f3-5188-4806-b788-0893a61634ed/93677179-2d99-4ef4-9e17-fe70d49abfbf/content.m3u8',
269 'formats': 'mincount:8',
273 'skip_download': True,
275 'skip': 'video gone',
277 # m3u8 served with Content-Type: text/plain
279 'url': 'http://www.nacentapps.com/m3u8/index.m3u8',
284 'upload_date': '20140720',
285 'formats': 'mincount:11',
289 'skip_download': True,
291 'skip': 'video gone',
295 'url': 'http://www.google.com/url?sa=t&rct=j&q=&esrc=s&source=web&cd=1&cad=rja&ved=0CCUQtwIwAA&url=http%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DcmQHVoWB5FY&ei=F-sNU-LLCaXk4QT52ICQBQ&usg=AFQjCNEw4hL29zgOohLXvpJ-Bdh2bils1Q&bvm=bv.61965928,d.bGE',
299 'upload_date': '20130224',
300 'uploader_id': 'TheVerge',
301 'description': 're:^Chris Ziegler takes a look at the\.*',
302 'uploader': 'The Verge',
303 'title': 'First Firefox OS phones side-by-side',
306 'skip_download': False,
310 # redirect in Refresh HTTP header
311 'url': 'https://www.facebook.com/l.php?u=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DpO8h3EaFRdo&h=TAQHsoToz&enc=AZN16h-b6o4Zq9pZkCCdOLNKMN96BbGMNtcFwHSaazus4JHT_MFYkAA-WARTX2kvsCIdlAIyHZjl6d33ILIJU7Jzwk_K3mcenAXoAzBNoZDI_Q7EXGDJnIhrGkLXo_LJ_pAa2Jzbx17UHMd3jAs--6j2zaeto5w9RTn8T_1kKg3fdC5WPX9Dbb18vzH7YFX0eSJmoa6SP114rvlkw6pkS1-T&s=1',
315 'title': 'Tripeo Boiler Room x Dekmantel Festival DJ Set',
316 'description': 'md5:6294cc1af09c4049e0652b51a2df10d5',
317 'upload_date': '20150917',
318 'uploader_id': 'brtvofficial',
319 'uploader': 'Boiler Room',
322 'skip_download': False,
326 'url': 'http://www.hodiho.fr/2013/02/regis-plante-sa-jeep.html',
327 'md5': '85b90ccc9d73b4acd9138d3af4c27f89',
329 'id': '13601338388002',
331 'uploader': 'www.hodiho.fr',
332 'title': 'R\u00e9gis plante sa Jeep',
335 # bandcamp page with custom domain
337 'add_ie': ['Bandcamp'],
338 'url': 'http://bronyrock.com/track/the-pony-mash',
342 'title': 'The Pony Mash',
343 'uploader': 'M_Pallante',
345 'skip': 'There is a limit of 200 free downloads / month for the test song',
347 # embedded brightcove video
348 # it also tests brightcove videos that need to set the 'Referer' in the
351 'add_ie': ['BrightcoveLegacy'],
352 'url': 'http://www.bfmtv.com/video/bfmbusiness/cours-bourse/cours-bourse-l-analyse-technique-154522/',
354 'id': '2765128793001',
356 'title': 'Le cours de bourse : l’analyse technique',
357 'description': 'md5:7e9ad046e968cb2d1114004aba466fd9',
358 'uploader': 'BFM BUSINESS',
361 'skip_download': True,
365 # https://github.com/rg3/youtube-dl/issues/2253
366 'url': 'http://bcove.me/i6nfkrc3',
367 'md5': '0ba9446db037002366bab3b3eb30c88c',
369 'id': '3101154703001',
371 'title': 'Still no power',
372 'uploader': 'thestar.com',
373 'description': 'Mississauga resident David Farmer is still out of power as a result of the ice storm a month ago. To keep the house warm, Farmer cuts wood from his property for a wood burning stove downstairs.',
375 'add_ie': ['BrightcoveLegacy'],
376 'skip': 'video gone',
379 'url': 'http://www.championat.com/video/football/v/87/87499.html',
380 'md5': 'fb973ecf6e4a78a67453647444222983',
382 'id': '3414141473001',
384 'title': 'Видео. Удаление Дзагоева (ЦСКА)',
385 'description': 'Онлайн-трансляция матча ЦСКА - "Волга"',
386 'uploader': 'Championat',
390 # https://github.com/rg3/youtube-dl/issues/3541
391 'add_ie': ['BrightcoveLegacy'],
392 'url': 'http://www.kijk.nl/sbs6/leermijvrouwenkennen/videos/jqMiXKAYan2S/aflevering-1',
394 'id': '3866516442001',
396 'title': 'Leer mij vrouwen kennen: Aflevering 1',
397 'description': 'Leer mij vrouwen kennen: Aflevering 1',
398 'uploader': 'SBS Broadcasting',
400 'skip': 'Restricted to Netherlands',
402 'skip_download': True, # m3u8 download
407 'url': 'http://www.rollingstone.com/music/videos/norwegian-dj-cashmere-cat-goes-spartan-on-with-me-premiere-20131219',
408 'md5': '166dd577b433b4d4ebfee10b0824d8ff',
410 'id': 'BwY2RxaTrTkslxOfcan0UCf0YqyvWysJ',
412 'title': '2cc213299525360.mov', # that's what we get
415 'add_ie': ['Ooyala'],
418 # ooyala video embedded with http://player.ooyala.com/iframe.js
419 'url': 'http://www.macrumors.com/2015/07/24/steve-jobs-the-man-in-the-machine-first-trailer/',
421 'id': 'p0MGJndjoG5SOKqO_hZJuZFPB-Tr5VgB',
423 'title': '"Steve Jobs: Man in the Machine" trailer',
424 'description': 'The first trailer for the Alex Gibney documentary "Steve Jobs: Man in the Machine."',
428 'skip_download': True,
430 'skip': 'movie expired',
434 'url': 'http://www.tested.com/science/weird/460206-tested-grinding-coffee-2000-frames-second/',
438 'title': 'Tested: Grinding Coffee at 2000 Frames Per Second',
439 'upload_date': '20140225',
440 'description': 'md5:06a40fbf30b220468f1e0957c0f558ff',
441 'uploader': 'Tested',
442 'uploader_id': 'testedcom',
444 # No need to test YoutubeIE here
446 'skip_download': True,
451 'url': 'http://www.theguardian.com/world/2014/mar/11/obama-zach-galifianakis-between-two-ferns',
455 'title': 'Between Two Ferns with Zach Galifianakis: President Barack Obama',
456 'description': 'Episode 18: President Barack Obama sits down with Zach Galifianakis for his most memorable interview yet.',
458 # HEAD requests lead to endless 301, while GET is OK
459 'expected_warnings': ['301'],
463 'url': 'http://www.rg.ru/2014/03/15/reg-dfo/anklav-anons.html',
467 'title': 'Охотское море стало целиком российским',
468 'description': 'md5:5ed62483b14663e2a95ebbe115eb8f43',
472 'skip_download': True,
477 'url': 'http://sch1298sz.mskobr.ru/dou_edu/karamel_ki/filial_galleries/video/iframe_src_http_tvc_ru_video_iframe_id_55304_isplay_false_acc_video_id_channel_brand_id_11_show_episodes_episode_id_32307_frameb/',
481 'title': 'Дошкольное воспитание',
486 'url': 'http://www.vestifinance.ru/articles/25753',
489 'title': 'Прямые трансляции с Форума-выставки "Госзаказ-2013"',
494 'title': 'Госзаказ. День 3',
500 'title': 'Госзаказ. День 2',
506 'title': 'Госзаказ. День 1',
512 'skip_download': True,
517 'url': 'http://www.kinomyvi.tv/news/detail/Pervij-dublirovannij-trejler--Uzhastikov-_nOw1',
519 'id': 'f4dafcad-ff21-423d-89b5-146cfd89fa1e',
521 'title': 'Ужастики, русский трейлер (2015)',
522 'thumbnail': 're:^https?://.*\.jpg$',
528 'url': 'http://www.numisc.com/forum/showthread.php?11696-FM15-which-pumiscer-was-this-%28-vid-%29-%28-alfa-as-fuck-srx-%29&s=711f5db534502e22260dec8c5e2d66d8',
531 'title': '[NSFL] [FM15] which pumiscer was this ( vid ) ( alfa as fuck srx )',
533 'playlist_mincount': 7,
534 # This forum does not allow <iframe> syntaxes anymore
535 # Now HTML tags are displayed as-is
536 'skip': 'No videos on this page',
540 'url': 'http://en.support.wordpress.com/videos/ted-talks/',
541 'md5': '65fdff94098e4a607385a60c5177c638',
545 'title': 'Hidden miracles of the natural world',
546 'uploader': 'Louie Schwartzberg',
547 'description': 'md5:8145d19d320ff3e52f28401f4c4283b9',
550 # Embedded Ustream video
552 'url': 'http://www.american.edu/spa/pti/nsa-privacy-janus-2014.cfm',
553 'md5': '27b99cdb639c9b12a79bca876a073417',
557 'uploader': 'AU SPA: The NSA and Privacy',
558 'title': 'NSA and Privacy Forum Debate featuring General Hayden and Barton Gellman'
561 # nowvideo embed hidden behind percent encoding
563 'url': 'http://www.waoanime.tv/the-super-dimension-fortress-macross-episode-1/',
564 'md5': '2baf4ddd70f697d94b1c18cf796d5107',
566 'id': '06e53103ca9aa',
568 'title': 'Macross Episode 001 Watch Macross Episode 001 onl',
569 'description': 'No description',
574 'url': 'http://www.tv-replay.fr/redirection/20-03-14/x-enius-arte-10753389.html',
575 'md5': '7653032cbb25bf6c80d80f217055fa43',
577 'id': '048195-004_PLUS7-F',
580 'description': 'md5:d5fdf32ef6613cdbfd516ae658abf168',
581 'upload_date': '20140320',
584 'skip_download': 'Requires rtmpdump'
586 'skip': 'video gone',
590 'url': 'http://www.tsprod.com/replay-du-concert-alcaline-de-calogero',
594 'title': 'Alcaline, le concert avec Calogero',
595 'description': 'md5:61f08036dcc8f47e9cfc33aed08ffaff',
596 'upload_date': '20150226',
597 'timestamp': 1424989860,
602 'skip_download': True,
604 'expected_warnings': [
610 'url': 'http://www.wired.com/2014/04/honda-asimo/',
611 'md5': 'ba0dfe966fa007657bd1443ee672db0f',
613 'id': '53501be369702d3275860000',
615 'title': 'Honda’s New Asimo Robot Is More Human Than Ever',
620 'url': 'http://www.spi0n.com/zap-spi0n-com-n216/',
621 'md5': '441aeeb82eb72c422c7f14ec533999cd',
623 'id': 'k2mm4bCdJ6CQ2i7c8o2',
625 'title': 'Le Zap de Spi0n n°216 - Zapping du Web',
626 'description': 'md5:faf028e48a461b8b7fad38f1e104b119',
628 'uploader_id': 'xgditw',
629 'upload_date': '20140425',
630 'timestamp': 1398441542,
632 'add_ie': ['Dailymotion'],
636 'url': 'http://www.badzine.de/ansicht/datum/2014/06/09/so-funktioniert-die-neue-englische-badminton-liga.html',
640 'title': 'The NBL Auction 2014',
641 'uploader': 'BADMINTON England',
642 'uploader_id': 'BADMINTONEvents',
643 'upload_date': '20140603',
644 'description': 'md5:9ef128a69f1e262a700ed83edb163a73',
646 'add_ie': ['Youtube'],
648 'skip_download': True,
653 'url': 'http://www.vulture.com/2016/06/new-key-peele-sketches-released.html',
654 'md5': 'ca1aef97695ef2c1d6973256a57e5252',
656 'id': '769f7ec0-0692-4d62-9b45-0d88074bffc1',
658 'title': 'Key and Peele|October 10, 2012|2|203|Liam Neesons - Uncensored',
659 'description': 'Two valets share their love for movie star Liam Neesons.',
660 'timestamp': 1349922600,
661 'upload_date': '20121011',
664 # YouTube embed via <data-embed-url="">
666 'url': 'https://play.google.com/store/apps/details?id=com.gameloft.android.ANMP.GloftA8HM',
670 'title': 'Asphalt 8: Airborne - Update - Welcome to Dubai!',
671 'uploader': 'Gameloft',
672 'uploader_id': 'gameloft',
673 'upload_date': '20140828',
674 'description': 'md5:c80da9ed3d83ae6d1876c834de03e1c4',
677 'skip_download': True,
682 'url': 'http://www.ll.mit.edu/workshops/education/videocourses/antennas/lecture1/video/',
684 'md5': '0c5e352edabf715d762b0ad4e6d9ee67',
686 'id': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final',
687 'title': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final - video1',
692 'md5': '10e4bb3aaca9fd630e273ff92d9f3c63',
694 'id': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final_PIP',
695 'title': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final - pip',
701 'title': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final',
706 'url': 'http://www.handjobhub.com/video/busty-blonde-siri-tit-fuck-while-wank-6313.html',
707 'md5': '9d65602bf31c6e20014319c7d07fba27',
709 'id': '5123ea6d5e5a7',
712 'uploader': 'www.handjobhub.com',
713 'title': 'Busty Blonde Siri Tit Fuck While Wank at HandjobHub.com',
716 # Multiple brightcove videos
717 # https://github.com/rg3/youtube-dl/issues/2283
719 'url': 'http://www.newyorker.com/online/blogs/newsdesk/2014/01/always-never-nuclear-command-and-control.html',
721 'id': 'always-never',
722 'title': 'Always / Never - The New Yorker',
726 'extract_flat': False,
727 'skip_download': True,
732 'url': 'http://umpire-empire.com/index.php/topic/58125-laz-decides-no-thats-low/',
733 'md5': '96f09a37e44da40dd083e12d9a683327',
737 'title': 'Ump changes call to ball',
738 'description': 'md5:71c11215384298a172a6dcb4c2e20685',
740 'timestamp': 1401537900,
741 'upload_date': '20140531',
742 'thumbnail': 're:^https?://.*\.jpg$',
747 'url': 'http://study.com/academy/lesson/north-american-exploration-failed-colonies-of-spain-france-england.html#lesson',
748 'md5': '1953f3a698ab51cfc948ed3992a0b7ff',
752 'title': 'paywall_north-american-exploration-failed-colonies-of-spain-france-england',
753 'description': 'a Paywall Videos video from Remilon',
755 'uploader': 'study.com',
756 'timestamp': 1459678540,
757 'upload_date': '20160403',
758 'filesize': 24687186,
762 'url': 'http://thoughtworks.wistia.com/medias/uxjb0lwrcz',
763 'md5': 'baf49c2baa8a7de5f3fc145a8506dcd4',
767 'title': 'Conversation about Hexagonal Rails Part 1',
768 'description': 'a Martin Fowler video from ThoughtWorks',
770 'uploader': 'thoughtworks.wistia.com',
771 'timestamp': 1401832161,
772 'upload_date': '20140603',
775 # Wistia standard embed (async)
777 'url': 'https://www.getdrip.com/university/brennan-dunn-drip-workshop/',
781 'title': 'Drip Brennan Dunn Workshop',
782 'description': 'a JV Webinars video from getdrip-1',
784 'timestamp': 1463607249,
785 'upload_date': '20160518',
788 'skip_download': True,
793 'url': 'http://nakedsecurity.sophos.com/2014/10/29/sscc-171-are-you-sure-that-1234-is-a-bad-password-podcast/',
797 'description': 'md5:ff867d6b555488ad3c52572bb33d432c',
798 'uploader': 'Sophos Security',
799 'title': 'Chet Chat 171 - Oct 29, 2014',
800 'upload_date': '20141029',
803 # Soundcloud multiple embeds
805 'url': 'http://www.guitarplayer.com/lessons/1014/legato-workout-one-hour-to-more-fluid-performance---tab/52809',
808 'title': 'Guitar Essentials: Legato Workout—One-Hour to Fluid Performance | TAB + AUDIO',
810 'playlist_mincount': 7,
814 'url': 'http://www.esa.int/Our_Activities/Space_Science/Rosetta/Philae_comet_touch-down_webcast',
818 'upload_date': '20141112',
819 'title': 'Rosetta #CometLanding webcast HL 10',
822 # Another Livestream embed, without 'new.' in URL
824 'url': 'https://www.freespeech.org/',
828 'title': 're:^FSTV [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
832 'skip_download': True,
837 'url': 'http://discourse.ubuntu.com/t/unity-8-desktop-mode-windows-on-mir/1986',
840 'title': 'Unity 8 desktop-mode windows on Mir! - Ubuntu Discourse',
842 'playlist_mincount': 2,
846 'url': 'http://undergroundwellness.com/podcasts/306-5-steps-to-permanent-gut-healing/',
850 'upload_date': '20141126',
851 'title': 'Jack Tips: 5 Steps to Permanent Gut Healing',
856 'url': 'http://www.abc.net.au/7.30/content/2015/s4164797.htm',
858 'id': '730m_DandD_1901_512k',
860 'uploader': 'www.abc.net.au',
861 'title': 'Game of Thrones with dice - Dungeons and Dragons fantasy role-playing game gets new life - 19/01/2015',
864 # embedded viddler video
866 'url': 'http://deadspin.com/i-cant-stop-watching-john-wall-chop-the-nuggets-with-th-1681801597',
870 'uploader': 'deadspin',
871 'title': 'WALL-TO-GORTAT',
872 'timestamp': 1422285291,
873 'upload_date': '20150126',
875 'add_ie': ['Viddler'],
879 'url': 'http://thedailyshow.cc.com/podcast/episodetwelve',
883 'title': "The Daily Show Podcast without Jon Stewart - Episode 12: Bassem Youssef: Egypt's Jon Stewart",
884 'description': 'md5:601cb790edd05908957dae8aaa866465',
885 'upload_date': '20150220',
887 'skip': 'All The Daily Show URLs now redirect to http://www.cc.com/shows/',
891 'url': 'http://media.nationalarchives.gov.uk/index.php/webinar-using-discovery-national-archives-online-catalogue/',
895 'upload_date': '20150212',
896 'uploader': 'The National Archives UK',
897 'description': 'md5:a236581cd2449dd2df4f93412f3f01c6',
898 'uploader_id': 'NationalArchives08',
899 'title': 'Webinar: Using Discovery, The National Archives’ online catalogue',
904 'url': 'http://www.rtlnieuws.nl/nieuws/buitenland/aanslagen-kopenhagen',
905 'playlist_mincount': 5,
907 'id': 'aanslagen-kopenhagen',
908 'title': 'Aanslagen Kopenhagen | RTL Nieuws',
913 'url': 'http://www.skipass.com/news/116090-bon-appetit-s5ep3-baqueira-mi-cor.html',
917 'title': 'EP3S5 - Bon Appétit - Baqueira Mi Corazon !',
920 # Kaltura embed (different embed code)
922 'url': 'http://www.premierchristianradio.com/Shows/Saturday/Unbelievable/Conference-Videos/Os-Guinness-Is-It-Fools-Talk-Unbelievable-Conference-2014',
926 'upload_date': '20150127',
927 'uploader_id': 'PremierMedia',
929 'title': 'Os Guinness // Is It Fools Talk? // Unbelievable? Conference 2014',
932 # Kaltura embed protected with referrer
934 'url': 'http://www.disney.nl/disney-channel/filmpjes/achter-de-schermen#/videoId/violetta-achter-de-schermen-ruggero',
938 'title': 'Violetta - Achter De Schermen - Ruggero',
939 'description': 'Achter de schermen met Ruggero',
940 'timestamp': 1435133761,
941 'upload_date': '20150624',
942 'uploader_id': 'echojecka',
945 # Kaltura embed with single quotes
947 'url': 'http://fod.infobase.com/p_ViewPlaylist.aspx?AssignmentID=NUN8ZY',
952 'timestamp': 1355743100,
953 'upload_date': '20121217',
954 'uploader_id': 'batchUser',
956 'add_ie': ['Kaltura'],
959 # Kaltura embedded via quoted entry_id
960 'url': 'https://www.oreilly.com/ideas/my-cloud-makes-pretty-pictures',
964 'title': '06_matthew_brender_raj_dutt',
965 'timestamp': 1466638791,
966 'upload_date': '20160622',
968 'add_ie': ['Kaltura'],
969 'expected_warnings': [
970 'Could not send HEAD request'
973 'skip_download': True,
977 # Kaltura embedded, some fileExt broken (#11480)
978 'url': 'http://www.cornell.edu/video/nima-arkani-hamed-standard-models-of-particle-physics',
982 'title': 'Our "Standard Models" of particle physics and cosmology',
983 'description': 'md5:67ea74807b8c4fea92a6f38d6d323861',
984 'timestamp': 1321158993,
985 'upload_date': '20111113',
986 'uploader_id': 'kps1',
988 'add_ie': ['Kaltura'],
990 # Eagle.Platform embed (generic URL)
992 'url': 'http://lenta.ru/news/2015/03/06/navalny/',
993 # Not checking MD5 as sometimes the direct HTTP link results in 404 and HLS is used
997 'title': 'Навальный вышел на свободу',
998 'description': 'md5:d97861ac9ae77377f3f20eaf9d04b4f5',
999 'thumbnail': 're:^https?://.*\.jpg$',
1005 # ClipYou (Eagle.Platform) embed (custom URL)
1007 'url': 'http://muz-tv.ru/play/7129/',
1008 # Not checking MD5 as sometimes the direct HTTP link results in 404 and HLS is used
1012 'title': "'O Sole Mio",
1013 'thumbnail': 're:^https?://.*\.jpg$',
1020 'url': 'http://muz-tv.ru/kinozal/view/7400/',
1024 'title': 'Тайны перевала Дятлова • 1 серия 2 часть',
1025 'description': 'Документальный сериал-расследование одной из самых жутких тайн ХХ века',
1026 'thumbnail': 're:^https?://.*\.jpg$',
1033 'url': 'http://www.cinemablend.com/new/First-Joe-Dirt-2-Trailer-Teaser-Stupid-Greatness-70874.html',
1037 'title': 'Joe Dirt 2 Beautiful Loser Teaser Trailer',
1038 'thumbnail': 're:^https?://.*\.png$',
1044 'url': 'http://techcrunch.com/video/facebook-creates-on-this-day-crunch-report/518726732/',
1045 'md5': '4c6f127a30736b59b3e2c19234ee2bf7',
1049 'title': 'Facebook Creates "On This Day" | Crunch Report',
1054 'url': 'http://www.svt.se/sport/ishockey/jagr-tacklar-giroux-under-intervjun',
1058 'title': 'Här trycker Jagr till Giroux (under SVT-intervjun)',
1063 # Crooks and Liars embed
1065 'url': 'http://crooksandliars.com/2015/04/fox-friends-says-protecting-atheists',
1069 'title': "Fox & Friends Says Protecting Atheists From Discrimination Is Anti-Christian!",
1070 'description': 'md5:e1a46ad1650e3a5ec7196d432799127f',
1071 'timestamp': 1428207000,
1072 'upload_date': '20150405',
1073 'uploader': 'Heather',
1076 # Crooks and Liars external embed
1078 'url': 'http://theothermccain.com/2010/02/02/video-proves-that-bill-kristol-has-been-watching-glenn-beck/comment-page-1/',
1080 'id': 'MTE3MjUtMzQ2MzA',
1082 'title': 'md5:5e3662a81a4014d24c250d76d41a08d5',
1083 'description': 'md5:9b8e9542d6c3c5de42d6451b7d780cec',
1084 'timestamp': 1265032391,
1085 'upload_date': '20100201',
1086 'uploader': 'Heather',
1089 # NBC Sports vplayer embed
1091 'url': 'http://www.riderfans.com/forum/showthread.php?121827-Freeman&s=e98fa1ea6dc08e886b1678d35212494a',
1093 'id': 'ln7x1qSThw4k',
1095 'title': "PFT Live: New leader in the 'new-look' defense",
1096 'description': 'md5:65a19b4bbfb3b0c0c5768bed1dfad74e',
1097 'uploader': 'NBCU-SPORTS',
1098 'upload_date': '20140107',
1099 'timestamp': 1389118457,
1104 'url': 'http://www.vulture.com/2016/06/letterman-couldnt-care-less-about-late-night.html',
1105 'md5': '1aa589c675898ae6d37a17913cf68d66',
1107 'id': '701714499682',
1109 'title': 'PREVIEW: On Assignment: David Letterman',
1110 'description': 'A preview of Tom Brokaw\'s interview with David Letterman as part of the On Assignment series powered by Dateline. Airs Sunday June 12 at 7/6c.',
1115 'url': 'https://video.udn.com/news/300346',
1116 'md5': 'fd2060e988c326991037b9aff9df21a6',
1120 'title': '中一中男師變性 全校師生力挺',
1121 'thumbnail': 're:^https?://.*\.jpg$',
1125 'skip_download': True,
1130 'url': 'http://www.businessinsider.com/excel-index-match-vlookup-video-how-to-2015-2?IR=T',
1132 'id': '50YnY4czr4ms1vJ7yz3xzq0excz_pUMs',
1134 'description': 'VIDEO: INDEX/MATCH versus VLOOKUP.',
1135 'title': 'This is what separates the Excel masters from the wannabes',
1136 'duration': 191.933,
1140 'skip_download': True,
1143 # Brightcove URL in single quotes
1145 'url': 'http://www.sportsnet.ca/baseball/mlb/sn-presents-russell-martin-world-citizen/',
1146 'md5': '4ae374f1f8b91c889c4b9203c8c752af',
1148 'id': '4255764656001',
1150 'title': 'SN Presents: Russell Martin, World Citizen',
1151 'description': 'To understand why he was the Toronto Blue Jays’ top off-season priority is to appreciate his background and upbringing in Montreal, where he first developed his baseball skills. Written and narrated by Stephen Brunt.',
1152 'uploader': 'Rogers Sportsnet',
1153 'uploader_id': '1704050871',
1154 'upload_date': '20150525',
1155 'timestamp': 1432570283,
1158 # Dailymotion Cloud video
1160 'url': 'http://replay.publicsenat.fr/vod/le-debat/florent-kolandjian,dominique-cena,axel-decourtye,laurence-abeille,bruno-parmentier/175910',
1161 'md5': 'dcaf23ad0c67a256f4278bce6e0bae38',
1165 'title': 'Sauvons les abeilles ! - Le débat',
1166 'description': 'md5:d9082128b1c5277987825d684939ca26',
1167 'thumbnail': 're:^https?://.*\.jpe?g$',
1168 'timestamp': 1434970506,
1169 'upload_date': '20150622',
1170 'uploader': 'Public Sénat',
1171 'uploader_id': 'xa9gza',
1174 # OnionStudios embed
1176 'url': 'http://www.clickhole.com/video/dont-understand-bitcoin-man-will-mumble-explanatio-2537',
1180 'title': 'Don’t Understand Bitcoin? This Man Will Mumble An Explanation At You',
1181 'thumbnail': 're:^https?://.*\.jpe?g$',
1182 'uploader': 'ClickHole',
1183 'uploader_id': 'clickhole',
1188 'url': 'http://whilewewatch.blogspot.ru/2012/06/whilewewatch-whilewewatch-gripping.html',
1190 'id': '74849a00-85a9-11e1-9660-123139220831',
1192 'title': '#whilewewatch',
1195 # AdobeTVVideo embed
1197 'url': 'https://helpx.adobe.com/acrobat/how-to/new-experience-acrobat-dc.html?set=acrobat--get-started--essential-beginners',
1198 'md5': '43662b577c018ad707a63766462b1e87',
1202 'title': 'New experience with Acrobat DC',
1203 'description': 'New experience with Acrobat DC',
1204 'duration': 248.667,
1207 # BrightcoveInPageEmbed embed
1209 'url': 'http://www.geekandsundry.com/tabletop-bonus-wils-final-thoughts-on-dread/',
1211 'id': '4238694884001',
1213 'title': 'Tabletop: Dread, Last Thoughts',
1214 'description': 'Tabletop: Dread, Last Thoughts',
1218 # Brightcove embed, with no valid 'renditions' but valid 'IOSRenditions'
1219 # This video can't be played in browsers if Flash disabled and UA set to iPhone, which is actually a false alarm
1221 'url': 'https://dl.dropboxusercontent.com/u/29092637/interview.html',
1223 'id': '4785848093001',
1225 'title': 'The Cardinal Pell Interview',
1226 'description': 'Sky News Contributor Andrew Bolt interviews George Pell in Rome, following the Cardinal\'s evidence before the Royal Commission into Child Abuse. ',
1227 'uploader': 'GlobeCast Australia - GlobeStream',
1228 'uploader_id': '2733773828001',
1229 'upload_date': '20160304',
1230 'timestamp': 1457083087,
1234 'skip_download': True,
1237 # Another form of arte.tv embed
1239 'url': 'http://www.tv-replay.fr/redirection/09-04-16/arte-reportage-arte-11508975.html',
1240 'md5': '850bfe45417ddf221288c88a0cffe2e2',
1242 'id': '030273-562_PLUS7-F',
1244 'title': 'ARTE Reportage - Nulle part, en France',
1245 'description': 'md5:e3a0e8868ed7303ed509b9e3af2b870d',
1246 'upload_date': '20160409',
1251 'url': 'http://www.wykop.pl/link/3088787/',
1252 'md5': 'ace83b9ed19b21f68e1b50e844fdf95d',
1254 'id': '874_1459135191',
1256 'title': 'Man shows poor quality of new apartment building',
1257 'description': 'The wall is like a sand pile.',
1258 'uploader': 'Lake8737',
1261 # Duplicated embedded video URLs
1263 'url': 'http://www.hudl.com/athlete/2538180/highlights/149298443',
1265 'id': '149298443_480_16c25b74_2',
1267 'title': 'vs. Blue Orange Spring Game',
1268 'uploader': 'www.hudl.com',
1271 # twitter:player:stream embed
1273 'url': 'http://www.rtl.be/info/video/589263.aspx?CategoryID=288',
1277 'title': 'Une nouvelle espèce de dinosaure découverte en Argentine',
1278 'uploader': 'www.rtl.be',
1282 'skip_download': True,
1285 # twitter:player embed
1287 'url': 'http://www.theatlantic.com/video/index/484130/what-do-black-holes-sound-like/',
1288 'md5': 'a3e0df96369831de324f0778e126653c',
1290 'id': '4909620399001',
1292 'title': 'What Do Black Holes Sound Like?',
1293 'description': 'what do black holes sound like',
1294 'upload_date': '20160524',
1295 'uploader_id': '29913724001',
1296 'timestamp': 1464107587,
1297 'uploader': 'TheAtlantic',
1299 'add_ie': ['BrightcoveLegacy'],
1301 # Facebook <iframe> embed
1303 'url': 'https://www.hostblogger.de/blog/archives/6181-Auto-jagt-Betonmischer.html',
1304 'md5': 'fbcde74f534176ecb015849146dd3aee',
1306 'id': '599637780109885',
1308 'title': 'Facebook video #599637780109885',
1311 # Facebook API embed
1313 'url': 'http://www.lothype.com/blue-stars-2016-preview-standstill-full-show/',
1314 'md5': 'a47372ee61b39a7b90287094d447d94e',
1316 'id': '10153467542406923',
1318 'title': 'Facebook video #10153467542406923',
1321 # Wordpress "YouTube Video Importer" plugin
1323 'url': 'http://www.lothype.com/blue-devils-drumline-stanford-lot-2016/',
1324 'md5': 'd16797741b560b485194eddda8121b48',
1326 'id': 'HNTXWDXV9Is',
1328 'title': 'Blue Devils Drumline Stanford lot 2016',
1329 'upload_date': '20160627',
1330 'uploader_id': 'GENOCIDE8GENERAL10',
1331 'uploader': 'cylus cyrus',
1335 # video stored on custom kaltura server
1336 'url': 'http://www.expansion.com/multimedia/videos.html?media=EQcM30NHIPv',
1337 'md5': '537617d06e64dfed891fa1593c4b30cc',
1341 'title': 'Elecciones británicas: 5 lecciones para Rajoy',
1342 'description': 'md5:435a89d68b9760b92ce67ed227055f16',
1343 'uploader_id': 'videos.expansion@el-mundo.net',
1344 'upload_date': '20150429',
1345 'timestamp': 1430303472,
1347 'add_ie': ['Kaltura'],
1350 # Non-standard Vimeo embed
1351 'url': 'https://openclassrooms.com/courses/understanding-the-web',
1352 'md5': '64d86f1c7d369afd9a78b38cbb88d80a',
1356 'title': 'Understanding the web - Teaser',
1357 'description': 'This is "Understanding the web - Teaser" by openclassrooms on Vimeo, the home for high quality videos and the people who love them.',
1358 'upload_date': '20151214',
1359 'uploader': 'OpenClassrooms',
1360 'uploader_id': 'openclassrooms',
1362 'add_ie': ['Vimeo'],
1365 # generic vimeo embed that requires original URL passed as Referer
1366 'url': 'http://racing4everyone.eu/2016/07/30/formula-1-2016-round12-germany/',
1367 'only_matching': True,
1370 'url': 'https://support.arkena.com/display/PLAY/Ways+to+embed+your+video',
1371 'md5': 'b96f2f71b359a8ecd05ce4e1daa72365',
1373 'id': 'b41dda37-d8e7-4d3f-b1b5-9a9db578bdfe',
1375 'title': 'Big Buck Bunny',
1376 'description': 'Royalty free test video',
1377 'timestamp': 1432816365,
1378 'upload_date': '20150528',
1382 'skip_download': True,
1384 'add_ie': [ArkenaIE
.ie_key()],
1387 'url': 'http://nova.bg/news/view/2016/08/16/156543/%D0%BD%D0%B0-%D0%BA%D0%BE%D1%81%D1%8A%D0%BC-%D0%BE%D1%82-%D0%B2%D0%B7%D1%80%D0%B8%D0%B2-%D0%BE%D1%82%D1%86%D0%B5%D0%BF%D0%B8%D1%85%D0%B0-%D1%86%D1%8F%D0%BB-%D0%BA%D0%B2%D0%B0%D1%80%D1%82%D0%B0%D0%BB-%D0%B7%D0%B0%D1%80%D0%B0%D0%B4%D0%B8-%D0%B8%D0%B7%D1%82%D0%B8%D1%87%D0%B0%D0%BD%D0%B5-%D0%BD%D0%B0-%D0%B3%D0%B0%D0%B7-%D0%B2-%D0%BF%D0%BB%D0%BE%D0%B2%D0%B4%D0%B8%D0%B2/',
1391 'title': 'НА КОСЪМ ОТ ВЗРИВ: Изтичане на газ на бензиностанция в Пловдив',
1394 'skip_download': True,
1396 'add_ie': [Vbox7IE
.ie_key()],
1400 'url': 'http://www.dagbladet.no/2016/02/23/nyheter/nordlys/ski/troms/ver/43254897/',
1403 'title': 'Etter ett års planlegging, klaffet endelig alt: - Jeg måtte ta en liten dans',
1405 'playlist_mincount': 3,
1408 # # TODO: find another test
1409 # # http://schema.org/VideoObject
1410 # 'url': 'https://flipagram.com/f/nyvTSJMKId',
1411 # 'md5': '888dcf08b7ea671381f00fab74692755',
1413 # 'id': 'nyvTSJMKId',
1415 # 'title': 'Flipagram by sjuria101 featuring Midnight Memories by One Direction',
1416 # 'description': '#love for cats.',
1417 # 'timestamp': 1461244995,
1418 # 'upload_date': '20160421',
1421 # 'force_generic_extractor': True,
1426 def report_following_redirect(self
, new_url
):
1427 """Report information extraction."""
1428 self
._downloader
.to_screen('[redirect] Following redirect to %s' % new_url
)
1430 def _extract_rss(self
, url
, video_id
, doc
):
1431 playlist_title
= doc
.find('./channel/title').text
1432 playlist_desc_el
= doc
.find('./channel/description')
1433 playlist_desc
= None if playlist_desc_el
is None else playlist_desc_el
.text
1436 for it
in doc
.findall('./channel/item'):
1437 next_url
= xpath_text(it
, 'link', fatal
=False)
1439 enclosure_nodes
= it
.findall('./enclosure')
1440 for e
in enclosure_nodes
:
1441 next_url
= e
.attrib
.get('url')
1451 'title': it
.find('title').text
,
1455 '_type': 'playlist',
1457 'title': playlist_title
,
1458 'description': playlist_desc
,
1462 def _extract_camtasia(self
, url
, video_id
, webpage
):
1463 """ Returns None if no camtasia video can be found. """
1465 camtasia_cfg
= self
._search
_regex
(
1466 r
'fo\.addVariable\(\s*"csConfigFile",\s*"([^"]+)"\s*\);',
1467 webpage
, 'camtasia configuration file', default
=None)
1468 if camtasia_cfg
is None:
1471 title
= self
._html
_search
_meta
('DC.title', webpage
, fatal
=True)
1473 camtasia_url
= compat_urlparse
.urljoin(url
, camtasia_cfg
)
1474 camtasia_cfg
= self
._download
_xml
(
1475 camtasia_url
, video_id
,
1476 note
='Downloading camtasia configuration',
1477 errnote
='Failed to download camtasia configuration')
1478 fileset_node
= camtasia_cfg
.find('./playlist/array/fileset')
1481 for n
in fileset_node
.getchildren():
1482 url_n
= n
.find('./uri')
1487 'id': os
.path
.splitext(url_n
.text
.rpartition('/')[2])[0],
1488 'title': '%s - %s' % (title
, n
.tag
),
1489 'url': compat_urlparse
.urljoin(url
, url_n
.text
),
1490 'duration': float_or_none(n
.find('./duration').text
),
1494 '_type': 'playlist',
1499 def _real_extract(self
, url
):
1500 if url
.startswith('//'):
1503 'url': self
.http_scheme() + url
,
1506 parsed_url
= compat_urlparse
.urlparse(url
)
1507 if not parsed_url
.scheme
:
1508 default_search
= self
._downloader
.params
.get('default_search')
1509 if default_search
is None:
1510 default_search
= 'fixup_error'
1512 if default_search
in ('auto', 'auto_warning', 'fixup_error'):
1514 self
._downloader
.report_warning('The url doesn\'t specify the protocol, trying with http')
1515 return self
.url_result('http://' + url
)
1516 elif default_search
!= 'fixup_error':
1517 if default_search
== 'auto_warning':
1518 if re
.match(r
'^(?:url|URL)$', url
):
1519 raise ExtractorError(
1520 'Invalid URL: %r . Call youtube-dl like this: youtube-dl -v "https://www.youtube.com/watch?v=BaW_jenozKc" ' % url
,
1523 self
._downloader
.report_warning(
1524 'Falling back to youtube search for %s . Set --default-search "auto" to suppress this warning.' % url
)
1525 return self
.url_result('ytsearch:' + url
)
1527 if default_search
in ('error', 'fixup_error'):
1528 raise ExtractorError(
1529 '%r is not a valid URL. '
1530 'Set --default-search "ytsearch" (or run youtube-dl "ytsearch:%s" ) to search YouTube'
1531 % (url
, url
), expected
=True)
1533 if ':' not in default_search
:
1534 default_search
+= ':'
1535 return self
.url_result(default_search
+ url
)
1537 url
, smuggled_data
= unsmuggle_url(url
)
1538 force_videoid
= None
1539 is_intentional
= smuggled_data
and smuggled_data
.get('to_generic')
1540 if smuggled_data
and 'force_videoid' in smuggled_data
:
1541 force_videoid
= smuggled_data
['force_videoid']
1542 video_id
= force_videoid
1544 video_id
= self
._generic
_id
(url
)
1546 self
.to_screen('%s: Requesting header' % video_id
)
1548 head_req
= HEADRequest(url
)
1549 head_response
= self
._request
_webpage
(
1551 note
=False, errnote
='Could not send HEAD request to %s' % url
,
1554 if head_response
is not False:
1555 # Check for redirect
1556 new_url
= head_response
.geturl()
1558 self
.report_following_redirect(new_url
)
1560 new_url
= smuggle_url(
1561 new_url
, {'force_videoid': force_videoid}
)
1562 return self
.url_result(new_url
)
1564 full_response
= None
1565 if head_response
is False:
1566 request
= sanitized_Request(url
)
1567 request
.add_header('Accept-Encoding', '*')
1568 full_response
= self
._request
_webpage
(request
, video_id
)
1569 head_response
= full_response
1573 'title': self
._generic
_title
(url
),
1574 'upload_date': unified_strdate(head_response
.headers
.get('Last-Modified'))
1577 # Check for direct link to a video
1578 content_type
= head_response
.headers
.get('Content-Type', '').lower()
1579 m
= re
.match(r
'^(?P<type>audio|video|application(?=/(?:ogg$|(?:vnd\.apple\.|x-)?mpegurl)))/(?P<format_id>[^;\s]+)', content_type
)
1581 format_id
= m
.group('format_id')
1582 if format_id
.endswith('mpegurl'):
1583 formats
= self
._extract
_m
3u8_formats
(url
, video_id
, 'mp4')
1584 elif format_id
== 'f4m':
1585 formats
= self
._extract
_f
4m
_formats
(url
, video_id
)
1588 'format_id': m
.group('format_id'),
1590 'vcodec': 'none' if m
.group('type') == 'audio' else None
1592 info_dict
['direct'] = True
1593 self
._sort
_formats
(formats
)
1594 info_dict
['formats'] = formats
1597 if not self
._downloader
.params
.get('test', False) and not is_intentional
:
1598 force
= self
._downloader
.params
.get('force_generic_extractor', False)
1599 self
._downloader
.report_warning(
1600 '%s on generic information extractor.' % ('Forcing' if force
else 'Falling back'))
1602 if not full_response
:
1603 request
= sanitized_Request(url
)
1604 # Some webservers may serve compressed content of rather big size (e.g. gzipped flac)
1605 # making it impossible to download only chunk of the file (yet we need only 512kB to
1606 # test whether it's HTML or not). According to youtube-dl default Accept-Encoding
1607 # that will always result in downloading the whole file that is not desirable.
1608 # Therefore for extraction pass we have to override Accept-Encoding to any in order
1609 # to accept raw bytes and being able to download only a chunk.
1610 # It may probably better to solve this by checking Content-Type for application/octet-stream
1611 # after HEAD request finishes, but not sure if we can rely on this.
1612 request
.add_header('Accept-Encoding', '*')
1613 full_response
= self
._request
_webpage
(request
, video_id
)
1615 first_bytes
= full_response
.read(512)
1617 # Is it an M3U playlist?
1618 if first_bytes
.startswith(b
'#EXTM3U'):
1619 info_dict
['formats'] = self
._extract
_m
3u8_formats
(url
, video_id
, 'mp4')
1620 self
._sort
_formats
(info_dict
['formats'])
1623 # Maybe it's a direct link to a video?
1624 # Be careful not to download the whole thing!
1625 if not is_html(first_bytes
):
1626 self
._downloader
.report_warning(
1627 'URL could be a direct video link, returning it as such.')
1634 webpage
= self
._webpage
_read
_content
(
1635 full_response
, url
, video_id
, prefix
=first_bytes
)
1637 self
.report_extraction(video_id
)
1639 # Is it an RSS feed, a SMIL file, an XSPF playlist or a MPD manifest?
1641 doc
= compat_etree_fromstring(webpage
.encode('utf-8'))
1642 if doc
.tag
== 'rss':
1643 return self
._extract
_rss
(url
, video_id
, doc
)
1644 elif doc
.tag
== 'SmoothStreamingMedia':
1645 info_dict
['formats'] = self
._parse
_ism
_formats
(doc
, url
)
1646 self
._sort
_formats
(info_dict
['formats'])
1648 elif re
.match(r
'^(?:{[^}]+})?smil$', doc
.tag
):
1649 smil
= self
._parse
_smil
(doc
, url
, video_id
)
1650 self
._sort
_formats
(smil
['formats'])
1652 elif doc
.tag
== '{http://xspf.org/ns/0/}playlist':
1653 return self
.playlist_result(self
._parse
_xspf
(doc
, video_id
), video_id
)
1654 elif re
.match(r
'(?i)^(?:{[^}]+})?MPD$', doc
.tag
):
1655 info_dict
['formats'] = self
._parse
_mpd
_formats
(
1657 mpd_base_url
=full_response
.geturl().rpartition('/')[0],
1659 self
._sort
_formats
(info_dict
['formats'])
1661 elif re
.match(r
'^{http://ns\.adobe\.com/f4m/[12]\.0}manifest$', doc
.tag
):
1662 info_dict
['formats'] = self
._parse
_f
4m
_formats
(doc
, url
, video_id
)
1663 self
._sort
_formats
(info_dict
['formats'])
1665 except compat_xml_parse_error
:
1668 # Is it a Camtasia project?
1669 camtasia_res
= self
._extract
_camtasia
(url
, video_id
, webpage
)
1670 if camtasia_res
is not None:
1673 # Sometimes embedded video player is hidden behind percent encoding
1674 # (e.g. https://github.com/rg3/youtube-dl/issues/2448)
1675 # Unescaping the whole page allows to handle those cases in a generic way
1676 webpage
= compat_urllib_parse_unquote(webpage
)
1678 # it's tempting to parse this further, but you would
1679 # have to take into account all the variations like
1680 # Video Title - Site Name
1681 # Site Name | Video Title
1682 # Video Title - Tagline | Site Name
1683 # and so on and so forth; it's just not practical
1684 video_title
= self
._og
_search
_title
(
1685 webpage
, default
=None) or self
._html
_search
_regex
(
1686 r
'(?s)<title>(.*?)</title>', webpage
, 'video title',
1689 # Try to detect age limit automatically
1690 age_limit
= self
._rta
_search
(webpage
)
1691 # And then there are the jokers who advertise that they use RTA,
1692 # but actually don't.
1693 AGE_LIMIT_MARKERS
= [
1694 r
'Proudly Labeled <a href="http://www.rtalabel.org/" title="Restricted to Adults">RTA</a>',
1696 if any(re
.search(marker
, webpage
) for marker
in AGE_LIMIT_MARKERS
):
1699 # video uploader is domain name
1700 video_uploader
= self
._search
_regex
(
1701 r
'^(?:https?://)?([^/]*)/.*', url
, 'video uploader')
1703 video_description
= self
._og
_search
_description
(webpage
, default
=None)
1704 video_thumbnail
= self
._og
_search
_thumbnail
(webpage
, default
=None)
1707 def _playlist_from_matches(matches
, getter
=None, ie
=None):
1709 self
.url_result(self
._proto
_relative
_url
(getter(m
) if getter
else m
), ie
)
1711 return self
.playlist_result(
1712 urlrs
, playlist_id
=video_id
, playlist_title
=video_title
)
1714 # Look for Brightcove Legacy Studio embeds
1715 bc_urls
= BrightcoveLegacyIE
._extract
_brightcove
_urls
(webpage
)
1717 self
.to_screen('Brightcove video detected.')
1720 'url': smuggle_url(bc_url
, {'Referer': url}
),
1721 'ie_key': 'BrightcoveLegacy'
1722 } for bc_url
in bc_urls
]
1725 '_type': 'playlist',
1726 'title': video_title
,
1731 # Look for Brightcove New Studio embeds
1732 bc_urls
= BrightcoveNewIE
._extract
_urls
(webpage
)
1734 return _playlist_from_matches(bc_urls
, ie
='BrightcoveNew')
1736 # Look for ThePlatform embeds
1737 tp_urls
= ThePlatformIE
._extract
_urls
(webpage
)
1739 return _playlist_from_matches(tp_urls
, ie
='ThePlatform')
1741 # Look for Vessel embeds
1742 vessel_urls
= VesselIE
._extract
_urls
(webpage
)
1744 return _playlist_from_matches(vessel_urls
, ie
=VesselIE
.ie_key())
1746 # Look for embedded rtl.nl player
1747 matches
= re
.findall(
1748 r
'<iframe[^>]+?src="((?:https?:)?//(?:www\.)?rtl\.nl/system/videoplayer/[^"]+(?:video_)?embed[^"]+)"',
1751 return _playlist_from_matches(matches
, ie
='RtlNl')
1753 vimeo_urls
= VimeoIE
._extract
_urls
(url
, webpage
)
1755 return _playlist_from_matches(vimeo_urls
, ie
=VimeoIE
.ie_key())
1757 vid_me_embed_url
= self
._search
_regex
(
1758 r
'src=[\'"](https?://vid\.me/[^\'"]+)[\'"]',
1759 webpage, 'vid.me embed', default=None)
1760 if vid_me_embed_url is not None:
1761 return self.url_result(vid_me_embed_url, 'Vidme')
1763 # Look for embedded YouTube player
1764 matches = re.findall(r'''(?x)
1773 (?P
<url
>(?
:https?
:)?
//(?
:www\
.)?
youtube(?
:-nocookie
)?\
.com
/
1777 return _playlist_from_matches(
1778 matches, lambda m: unescapeHTML(m[1]))
1780 # Look for lazyYT YouTube embed
1781 matches = re.findall(
1782 r'class="lazyYT" data-youtube-id="([^"]+)"', webpage)
1784 return _playlist_from_matches(matches, lambda m: unescapeHTML(m))
1786 # Look for Wordpress "YouTube Video Importer" plugin
1787 matches = re.findall(r'''(?x
)<div
[^
>]+
1788 class=(?P
<q1
>[\'"])[^\'"]*\byvii
_single
_video
_player
\b[^
\'"]*(?P=q1)[^>]+
1789 data-video_id=(?P<q2>[\'"])([^
\'"]+)(?P=q2)''', webpage)
1791 return _playlist_from_matches(matches, lambda m: m[-1])
1793 matches = DailymotionIE._extract_urls(webpage)
1795 return _playlist_from_matches(matches)
1797 # Look for embedded Dailymotion playlist player (#3822)
1799 r'<iframe[^>]+?src=(["\'])(?P
<url
>(?
:https?
:)?
//(?
:www\
.)?dailymotion\
.[a
-z
]{2,3}
/widget
/jukebox
\?.+?
)\
1', webpage)
1801 playlists = re.findall(
1802 r'list\
[\
]=/playlist
/([^
/]+)/', unescapeHTML(m.group('url
')))
1804 return _playlist_from_matches(
1805 playlists, lambda p: '//dailymotion
.com
/playlist
/%s' % p)
1807 # Look for embedded Wistia player
1809 r'<(?
:meta
[^
>]+?content|iframe
[^
>]+?src
)=(["\'])(?P<url>(?:https?:)?//(?:fast\.)?wistia\.net/embed/iframe/.+?)\1', webpage)
1811 embed_url = self._proto_relative_url(
1812 unescapeHTML(match.group('url')))
1814 '_type': 'url_transparent',
1817 'uploader': video_uploader,
1820 match = re.search(r'(?:id=["\']wistia_|data
-wistia
-?
id=["\']|Wistia\.embed\(["\'])(?P
<id>[^
"\']+)', webpage)
1823 '_type': 'url_transparent',
1824 'url': 'wistia:%s' % match.group('id'),
1826 'uploader': video_uploader,
1831 <script[^>]+src=(["'])(?:https?:)?//fast\.wistia\.com/assets/external/E-v1\.js\1[^>]*>.*?
1832 <div[^>]+class=(["']).*?
\bwistia
_async
_(?P
<id>[a
-z0
-9]+)\b.*?\
2
1835 return self.url_result(self._proto_relative_url(
1836 'wistia:%s' % match.group('id')), 'Wistia')
1838 # Look for SVT player
1839 svt_url = SVTIE._extract_url(webpage)
1841 return self.url_result(svt_url, 'SVT')
1843 # Look for embedded condenast player
1844 matches = re.findall(
1845 r'<iframe\s+(?:[a-zA-Z-]+="[^"]+"\s+)*?src="(https?://player\.cnevids\.com/embed/[^"]+")',
1849 '_type': 'playlist',
1852 'ie_key': 'CondeNast',
1854 } for ma in matches],
1855 'title': video_title,
1859 # Look for Bandcamp pages with custom domain
1860 mobj = re.search(r'<meta property="og:url"[^>]*?content="(.*?bandcamp\.com.*?)"', webpage)
1861 if mobj is not None:
1862 burl = unescapeHTML(mobj.group(1))
1863 # Don't set the extractor because it can be a track url or an album
1864 return self.url_result(burl)
1866 # Look for embedded Vevo player
1868 r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:cache\.)?vevo\.com/.+?)\1', webpage)
1869 if mobj is not None:
1870 return self.url_result(mobj.group('url'))
1872 # Look for embedded Viddler player
1874 r'<(?:iframe[^>]+?src|param[^>]+?value)=(["\'])(?P<url>(?:https?:)?//(?:www\.)?viddler\.com/(?:embed|player)/.+?)\1',
1876 if mobj is not None:
1877 return self.url_result(mobj.group('url'))
1879 # Look for NYTimes player
1881 r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//graphics8\.nytimes\.com/bcvideo/[^/]+/iframe/embed\.html.+?)\1>',
1883 if mobj is not None:
1884 return self.url_result(mobj.group('url'))
1886 # Look for Libsyn player
1888 r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//html5-player\.libsyn\.com/embed/.+?)\1', webpage)
1889 if mobj is not None:
1890 return self.url_result(mobj.group('url'))
1892 # Look for Ooyala videos
1893 mobj = (re.search(r'player\.ooyala\.com/[^"?]+[?#][^"]*?(?:embedCode|ec)=(?P<ec>[^"&]+)', webpage) or
1894 re.search(r'OO\.Player\.create\([\'"].*?[\'"],\s*[\'"](?P<ec>.{32})[\'"]', webpage) or
1895 re.search(r'SBN\.VideoLinkset\.ooyala\([\'"](?P<ec>.{32})[\'"]\)', webpage) or
1896 re.search(r'data-ooyala-video-id\s*=\s*[\'"](?P<ec>.{32})[\'"]', webpage))
1897 if mobj is not None:
1898 return OoyalaIE._build_url_result(smuggle_url(mobj.group('ec'), {'domain': url}))
1900 # Look for multiple Ooyala embeds on SBN network websites
1901 mobj = re.search(r'SBN\.VideoLinkset\.entryGroup\((\[.*?\])', webpage)
1902 if mobj is not None:
1903 embeds = self._parse_json(mobj.group(1), video_id, fatal=False)
1905 return _playlist_from_matches(
1906 embeds, getter=lambda v: OoyalaIE._url_for_embed_code(smuggle_url(v['provider_video_id'], {'domain': url})), ie='Ooyala')
1908 # Look for Aparat videos
1909 mobj = re.search(r'<iframe .*?src="(http://www\.aparat\.com/video/[^"]+)"', webpage)
1910 if mobj is not None:
1911 return self.url_result(mobj.group(1), 'Aparat')
1913 # Look for MPORA videos
1914 mobj = re.search(r'<iframe .*?src="(http://mpora\.(?:com|de)/videos/[^"]+)"', webpage)
1915 if mobj is not None:
1916 return self.url_result(mobj.group(1), 'Mpora')
1918 # Look for embedded NovaMov-based player
1920 r'''(?x
)<(?
:pagespeed_
)?iframe
[^
>]+?src
=(["\'])
1921 (?P<url>http://(?:(?:embed|www)\.)?
1923 nowvideo\.(?:ch|sx|eu|at|ag|co)|
1924 videoweed\.(?:es|com)|
1925 movshare\.(?:net|sx|ag)|
1926 divxstage\.(?:eu|net|ch|co|at|ag))
1927 /embed\.php.+?)\1''', webpage)
1928 if mobj is not None:
1929 return self.url_result(mobj.group('url'))
1931 # Look for embedded Facebook player
1932 facebook_url = FacebookIE._extract_url(webpage)
1933 if facebook_url is not None:
1934 return self.url_result(facebook_url, 'Facebook')
1936 # Look for embedded VK player
1937 mobj = re.search(r'<iframe[^>]+?src=(["\'])(?P
<url
>https?
://vk\
.com
/video_ext\
.php
.+?
)\
1', webpage)
1938 if mobj is not None:
1939 return self.url_result(mobj.group('url
'), 'VK
')
1941 # Look for embedded Odnoklassniki player
1942 mobj = re.search(r'<iframe
[^
>]+?src
=(["\'])(?P<url>https?://(?:odnoklassniki|ok)\.ru/videoembed/.+?)\1', webpage)
1943 if mobj is not None:
1944 return self.url_result(mobj.group('url'), 'Odnoklassniki')
1946 # Look for embedded ivi player
1947 mobj = re.search(r'<embed[^>]+?src=(["\'])(?P
<url
>https?
://(?
:www\
.)?ivi\
.ru
/video
/player
.+?
)\
1', webpage)
1948 if mobj is not None:
1949 return self.url_result(mobj.group('url
'), 'Ivi
')
1951 # Look for embedded Huffington Post player
1953 r'<iframe
[^
>]+?src
=(["\'])(?P<url>https?://embed\.live\.huffingtonpost\.com/.+?)\1', webpage)
1954 if mobj is not None:
1955 return self.url_result(mobj.group('url'), 'HuffPost')
1958 mobj = re.search(r'class=["\']embedly
-card
["\'][^>]href=["\'](?P
<url
>[^
"\']+)', webpage)
1959 if mobj is not None:
1960 return self.url_result(mobj.group('url'))
1961 mobj = re.search(r'class=["\']embedly
-embed
["\'][^>]src=["\'][^
"\']*url=(?P<url>[^&]+)', webpage)
1962 if mobj is not None:
1963 return self.url_result(compat_urllib_parse_unquote(mobj.group('url')))
1965 # Look for funnyordie embed
1966 matches = re.findall(r'<iframe[^>]+?src="(https?
://(?
:www\
.)?funnyordie\
.com
/embed
/[^
"]+)"', webpage)
1968 return _playlist_from_matches(
1969 matches, getter=unescapeHTML, ie='FunnyOrDie
')
1971 # Look for BBC iPlayer embed
1972 matches = re.findall(r'setPlaylist\
("(https?://www\.bbc\.co\.uk/iplayer/[^/]+/[\da-z]{8})"\
)', webpage)
1974 return _playlist_from_matches(matches, ie='BBCCoUk
')
1976 # Look for embedded RUTV player
1977 rutv_url = RUTVIE._extract_url(webpage)
1979 return self.url_result(rutv_url, 'RUTV
')
1981 # Look for embedded TVC player
1982 tvc_url = TVCIE._extract_url(webpage)
1984 return self.url_result(tvc_url, 'TVC
')
1986 # Look for embedded SportBox player
1987 sportbox_urls = SportBoxEmbedIE._extract_urls(webpage)
1989 return _playlist_from_matches(sportbox_urls, ie='SportBoxEmbed
')
1991 # Look for embedded XHamster player
1992 xhamster_urls = XHamsterEmbedIE._extract_urls(webpage)
1994 return _playlist_from_matches(xhamster_urls, ie='XHamsterEmbed
')
1996 # Look for embedded TNAFlixNetwork player
1997 tnaflix_urls = TNAFlixNetworkEmbedIE._extract_urls(webpage)
1999 return _playlist_from_matches(tnaflix_urls, ie=TNAFlixNetworkEmbedIE.ie_key())
2001 # Look for embedded PornHub player
2002 pornhub_urls = PornHubIE._extract_urls(webpage)
2004 return _playlist_from_matches(pornhub_urls, ie=PornHubIE.ie_key())
2006 # Look for embedded DrTuber player
2007 drtuber_urls = DrTuberIE._extract_urls(webpage)
2009 return _playlist_from_matches(drtuber_urls, ie=DrTuberIE.ie_key())
2011 # Look for embedded RedTube player
2012 redtube_urls = RedTubeIE._extract_urls(webpage)
2014 return _playlist_from_matches(redtube_urls, ie=RedTubeIE.ie_key())
2016 # Look for embedded Tvigle player
2018 r'<iframe
[^
>]+?src
=(["\'])(?P<url>(?:https?:)?//cloud\.tvigle\.ru/video/.+?)\1', webpage)
2019 if mobj is not None:
2020 return self.url_result(mobj.group('url'), 'Tvigle')
2022 # Look for embedded TED player
2024 r'<iframe[^>]+?src=(["\'])(?P
<url
>https?
://embed(?
:-ssl
)?\
.ted\
.com
/.+?
)\
1', webpage)
2025 if mobj is not None:
2026 return self.url_result(mobj.group('url
'), 'TED
')
2028 # Look for embedded Ustream videos
2030 r'<iframe
[^
>]+?src
=(["\'])(?P<url>http://www\.ustream\.tv/embed/.+?)\1', webpage)
2031 if mobj is not None:
2032 return self.url_result(mobj.group('url'), 'Ustream')
2034 # Look for embedded arte.tv player
2036 r'<(?:script|iframe) [^>]*?src="(?P
<url
>http
://www\
.arte\
.tv
/(?
:playerv2
/embed|arte_vp
/index
)[^
"]+)"',
2038 if mobj is not None:
2039 return self.url_result(mobj.group('url
'), 'ArteTVEmbed
')
2041 # Look for embedded francetv player
2043 r'<iframe
[^
>]+?src
=(["\'])(?P<url>(?:https?://)?embed\.francetv\.fr/\?ue=.+?)\1',
2045 if mobj is not None:
2046 return self.url_result(mobj.group('url'))
2048 # Look for embedded smotri.com player
2049 smotri_url = SmotriIE._extract_url(webpage)
2051 return self.url_result(smotri_url, 'Smotri')
2053 # Look for embedded Myvi.ru player
2054 myvi_url = MyviIE._extract_url(webpage)
2056 return self.url_result(myvi_url)
2058 # Look for embedded soundcloud player
2059 soundcloud_urls = SoundcloudIE._extract_urls(webpage)
2061 return _playlist_from_matches(soundcloud_urls, getter=unescapeHTML, ie=SoundcloudIE.ie_key())
2063 # Look for embedded mtvservices player
2064 mtvservices_url = MTVServicesEmbeddedIE._extract_url(webpage)
2066 return self.url_result(mtvservices_url, ie='MTVServicesEmbedded')
2068 # Look for embedded yahoo player
2070 r'<iframe[^>]+?src=(["\'])(?P
<url
>https?
://(?
:screen|movies
)\
.yahoo\
.com
/.+?\
.html
\?format
=embed
)\
1',
2072 if mobj is not None:
2073 return self.url_result(mobj.group('url
'), 'Yahoo
')
2075 # Look for embedded sbs.com.au player
2079 <meta\s+property="og:video"\s+content=|
2082 (["\'])(?P<url>https?://(?:www\.)?sbs\.com\.au/ondemand/video/.+?)\1''',
2084 if mobj is not None:
2085 return self.url_result(mobj.group('url
'), 'SBS
')
2087 # Look for embedded Cinchcast player
2089 r'<iframe
[^
>]+?src
=(["\'])(?P<url>https?://player\.cinchcast\.com/.+?)\1',
2091 if mobj is not None:
2092 return self.url_result(mobj.group('url'), 'Cinchcast')
2095 r'<iframe[^>]+?src=(["\'])(?P
<url
>https?
://m(?
:lb
)?\
.mlb\
.com
/shared
/video
/embed
/embed\
.html
\?.+?
)\
1',
2099 r'data
-video
-link
=["\'](?P<url>http://m.mlb.com/video/[^"\']+)',
2101 if mobj is not None:
2102 return self.url_result(mobj.group('url
'), 'MLB
')
2105 r'<(?
:iframe|script
)[^
>]+?src
=(["\'])(?P<url>%s)\1' % CondeNastIE.EMBED_URL,
2107 if mobj is not None:
2108 return self.url_result(self._proto_relative_url(mobj.group('url'), scheme='http:'), 'CondeNast')
2111 r'<iframe[^>]+src="(?P
<url
>https?
://(?
:new\
.)?livestream\
.com
/[^
"]+/player[^"]+)"',
2113 if mobj is not None:
2114 return self.url_result(mobj.group('url'), 'Livestream')
2116 # Look for Zapiks embed
2118 r'<iframe[^>]+src="(?P
<url
>https?
://(?
:www\
.)?zapiks\
.fr
/index\
.php
\?.+?
)"', webpage)
2119 if mobj is not None:
2120 return self.url_result(mobj.group('url'), 'Zapiks')
2122 # Look for Kaltura embeds
2123 kaltura_url = KalturaIE._extract_url(webpage)
2125 return self.url_result(smuggle_url(kaltura_url, {'source_url': url}), KalturaIE.ie_key())
2127 # Look for Eagle.Platform embeds
2128 eagleplatform_url = EaglePlatformIE._extract_url(webpage)
2129 if eagleplatform_url:
2130 return self.url_result(eagleplatform_url, EaglePlatformIE.ie_key())
2132 # Look for ClipYou (uses Eagle.Platform) embeds
2134 r'<iframe[^>]+src="https?
://(?P
<host
>media\
.clipyou\
.ru
)/index
/player
\?.*\brecord
_id
=(?P
<id>\d
+).*"', webpage)
2135 if mobj is not None:
2136 return self.url_result('eagleplatform:%(host)s:%(id)s' % mobj.groupdict(), 'EaglePlatform')
2138 # Look for Pladform embeds
2139 pladform_url = PladformIE._extract_url(webpage)
2141 return self.url_result(pladform_url)
2143 # Look for Videomore embeds
2144 videomore_url = VideomoreIE._extract_url(webpage)
2146 return self.url_result(videomore_url)
2148 # Look for Webcaster embeds
2149 webcaster_url = WebcasterFeedIE._extract_url(self, webpage)
2151 return self.url_result(webcaster_url, ie=WebcasterFeedIE.ie_key())
2153 # Look for Playwire embeds
2155 r'<script[^>]+data-config=(["\'])(?P
<url
>(?
:https?
:)?
//config\
.playwire\
.com
/.+?
)\
1', webpage)
2156 if mobj is not None:
2157 return self.url_result(mobj.group('url
'))
2159 # Look for 5min embeds
2161 r'<meta
[^
>]+property="og:video"[^
>]+content
="https?://embed\.5min\.com/(?P<id>[0-9]+)/?', webpage)
2162 if mobj is not None:
2163 return self.url_result('5min:%s' % mobj.group('id'), 'FiveMin')
2165 # Look for Crooks and Liars embeds
2167 r'<(?:iframe[^>]+src|param[^>]+value)=(["\'])(?P
<url
>(?
:https?
:)?
//embed\
.crooksandliars\
.com
/(?
:embed|v
)/.+?
)\
1', webpage)
2168 if mobj is not None:
2169 return self.url_result(mobj.group('url
'))
2171 # Look for NBC Sports VPlayer embeds
2172 nbc_sports_url = NBCSportsVPlayerIE._extract_url(webpage)
2174 return self.url_result(nbc_sports_url, 'NBCSportsVPlayer
')
2176 # Look for NBC News embeds
2177 nbc_news_embed_url = re.search(
2178 r'<iframe
[^
>]+src
=(["\'])(?P<url>(?:https?:)?//www\.nbcnews\.com/widget/video-embed/[^"\']+)\
1', webpage)
2179 if nbc_news_embed_url:
2180 return self.url_result(nbc_news_embed_url.group('url
'), 'NBCNews
')
2182 # Look for Google Drive embeds
2183 google_drive_url = GoogleDriveIE._extract_url(webpage)
2184 if google_drive_url:
2185 return self.url_result(google_drive_url, 'GoogleDrive
')
2187 # Look for UDN embeds
2189 r'<iframe
[^
>]+src
="(?P<url>%s)"' % UDNEmbedIE._PROTOCOL_RELATIVE_VALID_URL, webpage)
2190 if mobj is not None:
2191 return self.url_result(
2192 compat_urlparse.urljoin(url, mobj.group('url
')), 'UDNEmbed
')
2194 # Look for Senate ISVP iframe
2195 senate_isvp_url = SenateISVPIE._search_iframe_url(webpage)
2197 return self.url_result(senate_isvp_url, 'SenateISVP
')
2199 # Look for Dailymotion Cloud videos
2200 dmcloud_url = DailymotionCloudIE._extract_dmcloud_url(webpage)
2202 return self.url_result(dmcloud_url, 'DailymotionCloud
')
2204 # Look for OnionStudios embeds
2205 onionstudios_url = OnionStudiosIE._extract_url(webpage)
2206 if onionstudios_url:
2207 return self.url_result(onionstudios_url)
2209 # Look for ViewLift embeds
2210 viewlift_url = ViewLiftEmbedIE._extract_url(webpage)
2212 return self.url_result(viewlift_url)
2214 # Look for JWPlatform embeds
2215 jwplatform_url = JWPlatformIE._extract_url(webpage)
2217 return self.url_result(jwplatform_url, 'JWPlatform
')
2219 # Look for Digiteka embeds
2220 digiteka_url = DigitekaIE._extract_url(webpage)
2222 return self.url_result(self._proto_relative_url(digiteka_url), DigitekaIE.ie_key())
2224 # Look for Arkena embeds
2225 arkena_url = ArkenaIE._extract_url(webpage)
2227 return self.url_result(arkena_url, ArkenaIE.ie_key())
2229 # Look for Piksel embeds
2230 piksel_url = PikselIE._extract_url(webpage)
2232 return self.url_result(piksel_url, PikselIE.ie_key())
2234 # Look for Limelight embeds
2235 mobj = re.search(r'LimelightPlayer\
.doLoad(Media|Channel|ChannelList
)\
(["\'](?P<id>[a-z0-9]{32})', webpage)
2239 'Channel': 'channel',
2240 'ChannelList': 'channel_list',
2242 return self.url_result('limelight:%s:%s' % (
2243 lm[mobj.group(1)], mobj.group(2)), 'Limelight%s' % mobj.group(1), mobj.group(2))
2247 <object[^>]+class=(["\'])LimelightEmbeddedPlayerFlash\
1[^
>]*>.*?
2249 name
=(["\'])flashVars\2[^>]+
2250 value=(["\'])(?
:(?
!\
3).)*mediaId
=(?P
<id>[a
-z0
-9]{32}
)
2253 return self.url_result('limelight:media:%s' % mobj.group('id'))
2255 # Look for AdobeTVVideo embeds
2257 r'<iframe[^>]+src=[\'"]((?:https?:)?//video\.tv\.adobe\.com/v/\d+[^"]+)[\'"]',
2259 if mobj is not None:
2260 return self.url_result(
2261 self._proto_relative_url(unescapeHTML(mobj.group(1))),
2264 # Look for Vine embeds
2266 r'<iframe[^>]+src=[\'"]((?:https?:)?//(?:www\.)?vine\.co/v/[^/]+/embed/(?:simple|postcard))',
2268 if mobj is not None:
2269 return self.url_result(
2270 self._proto_relative_url(unescapeHTML(mobj.group(1))), 'Vine')
2272 # Look for VODPlatform embeds
2274 r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?vod-platform\.net/[eE]mbed/.+?)\1',
2276 if mobj is not None:
2277 return self.url_result(
2278 self._proto_relative_url(unescapeHTML(mobj.group('url'))), 'VODPlatform')
2280 # Look for Mangomolo embeds
2282 r'''(?x
)<iframe
[^
>]+src
=(["\'])(?P<url>(?:https?:)?//(?:www\.)?admin\.mangomolo\.com/analytics/index\.php/customers/embed/
2284 video\?.*?\bid=(?P<video_id>\d+)|
2285 index\?.*?\bchannelid=(?P<channel_id>(?:[A-Za-z0-9+/=]|%2B|%2F|%3D)+)
2286 ).+?)\1''', webpage)
2287 if mobj is not None:
2289 '_type': 'url_transparent',
2290 'url': self._proto_relative_url(unescapeHTML(mobj.group('url'))),
2291 'title': video_title,
2292 'description': video_description,
2293 'thumbnail': video_thumbnail,
2294 'uploader': video_uploader,
2296 video_id = mobj.group('video_id')
2299 'ie_key': 'MangomoloVideo',
2304 'ie_key': 'MangomoloLive',
2305 'id': mobj.group('channel_id'),
2309 # Look for Instagram embeds
2310 instagram_embed_url = InstagramIE._extract_embed_url(webpage)
2311 if instagram_embed_url is not None:
2312 return self.url_result(
2313 self._proto_relative_url(instagram_embed_url), InstagramIE.ie_key())
2315 # Look for LiveLeak embeds
2316 liveleak_url = LiveLeakIE._extract_url(webpage)
2318 return self.url_result(liveleak_url, 'LiveLeak')
2320 # Look for 3Q SDN embeds
2321 threeqsdn_url = ThreeQSDNIE._extract_url(webpage)
2324 '_type': 'url_transparent',
2325 'ie_key': ThreeQSDNIE.ie_key(),
2326 'url': self._proto_relative_url(threeqsdn_url),
2327 'title': video_title,
2328 'description': video_description,
2329 'thumbnail': video_thumbnail,
2330 'uploader': video_uploader,
2333 # Look for VBOX7 embeds
2334 vbox7_url = Vbox7IE._extract_url(webpage)
2336 return self.url_result(vbox7_url, Vbox7IE.ie_key())
2338 # Look for DBTV embeds
2339 dbtv_urls = DBTVIE._extract_urls(webpage)
2341 return _playlist_from_matches(dbtv_urls, ie=DBTVIE.ie_key())
2343 # Looking for http://schema.org/VideoObject
2344 json_ld = self._search_json_ld(
2345 webpage, video_id, default={}, expected_type='VideoObject')
2346 if json_ld.get('url'):
2348 'title': video_title or info_dict['title'],
2349 'description': video_description,
2350 'thumbnail': video_thumbnail,
2351 'age_limit': age_limit
2353 info_dict.update(json_ld)
2356 # Look for HTML5 media
2357 entries = self._parse_html5_media_entries(url, webpage, video_id, m3u8_id='hls')
2359 for entry in entries:
2362 'title': video_title,
2364 self._sort_formats(entry['formats'])
2365 return self.playlist_result(entries)
2367 def check_video(vurl):
2368 if YoutubeIE.suitable(vurl):
2370 vpath = compat_urlparse.urlparse(vurl).path
2371 vext = determine_ext(vpath)
2372 return '.' in vpath and vext not in ('swf', 'png', 'jpg', 'srt', 'sbv', 'sub', 'vtt', 'ttml', 'js')
2374 def filter_video(urls):
2375 return list(filter(check_video, urls))
2377 # Start with something easy: JW Player in SWFObject
2378 found = filter_video(re.findall(r'flashvars: [\'"](?
:.*&)?
file=(http
[^
\'"&]*)', webpage))
2380 # Look for gorilla-vid style embedding
2381 found = filter_video(re.findall(r'''(?sx)
2385 jwplayer\s*\(\s*["'][^'"]+["']\s*\)\s*\.setup
2388 ['"]?file['"]?\s
*:\s
*["\'](.*?)["\']''', webpage))
2390 # Broaden the search a little bit
2391 found = filter_video(re.findall(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage))
2393 # Broaden the findall a little bit: JWPlayer JS loader
2394 found = filter_video(re.findall(
2395 r'[^A-Za-z0-9]?(?:file|video_url)["\']?:\s*["\'](http(?![^\'"]+\.[0-9]+[\'"])[^\'"]+)["\']', webpage))
2398 found = filter_video(re.findall(r'''(?xs
)
2399 flowplayer\
("[^"]+",\s*
2401 \s*\{[^}]+? ["']?clip["']?\s
*:\s
*\
{\s
*
2402 ["']?url["']?\s*:\s*["']([^
"']+)["']
2407 r"cinerama\.embedPlayer\(\s*\'[^']+\',\s
*'([^']+)'", webpage)
2409 # Try to find twitter cards info
2410 # twitter:player:stream should be checked before twitter:player since
2411 # it is expected to contain a raw stream (see
2412 # https://dev.twitter.com/cards/types/player#On_twitter.com_via_desktop_browser)
2413 found = filter_video(re.findall(
2414 r'<meta (?
:property|name
)="twitter:player:stream" (?
:content|value
)="(.+?)"', webpage))
2416 # We look for Open Graph info:
2417 # We have to match any number spaces between elements, some sites try to align them (eg.: statigr.am)
2418 m_video_type = re.findall(r'<meta
.*?
property="og:video:type".*?content
="video/(.*?)"', webpage)
2419 # We only look in og:video if the MIME type is a video, don't
try if it
's a Flash player:
2420 if m_video_type is not None:
2421 found = filter_video(re.findall(r'<meta
.*?
property="og:video".*?content
="(.*?)"', webpage))
2423 REDIRECT_REGEX = r'[0-9]{,2}
;\s
*(?
:URL|url
)=\'?
([^
\'"]+)'
2425 r'(?i)<meta\s+(?=(?:[a-z-]+="[^
"]+"\s
+)*http
-equiv
="refresh")'
2426 r'(?
:[a
-z
-]+="[^"]+"\s+)*?content="%s' % REDIRECT_REGEX,
2429 # Look also in Refresh HTTP header
2430 refresh_header = head_response.headers.get('Refresh
')
2432 # In python 2 response HTTP headers are bytestrings
2433 if sys.version_info < (3, 0) and isinstance(refresh_header, str):
2434 refresh_header = refresh_header.decode('iso
-8859-1')
2435 found = re.search(REDIRECT_REGEX, refresh_header)
2437 new_url = compat_urlparse.urljoin(url, unescapeHTML(found.group(1)))
2438 self.report_following_redirect(new_url)
2445 # twitter:player is a https URL to iframe player that may or may not
2446 # be supported by youtube-dl thus this is checked the very last (see
2447 # https://dev.twitter.com/cards/types/player#On_twitter.com_via_desktop_browser)
2448 embed_url = self._html_search_meta('twitter
:player
', webpage, default=None)
2450 return self.url_result(embed_url)
2453 raise UnsupportedError(url)
2456 for video_url in orderedSet(found):
2457 video_url = unescapeHTML(video_url)
2458 video_url = video_url.replace('\\/', '/')
2459 video_url = compat_urlparse.urljoin(url, video_url)
2460 video_id = compat_urllib_parse_unquote(os.path.basename(video_url))
2462 # Sometimes, jwplayer extraction will result in a YouTube URL
2463 if YoutubeIE.suitable(video_url):
2464 entries.append(self.url_result(video_url, 'Youtube
'))
2467 # here's a fun little line of code
for you
:
2468 video_id
= os
.path
.splitext(video_id
)[0]
2472 'uploader': video_uploader
,
2473 'title': video_title
,
2474 'age_limit': age_limit
,
2477 ext
= determine_ext(video_url
)
2479 entry_info_dict
['formats'] = self
._extract
_smil
_formats
(video_url
, video_id
)
2481 return self
.playlist_result(self
._extract
_xspf
_playlist
(video_url
, video_id
), video_id
)
2483 entry_info_dict
['formats'] = self
._extract
_m
3u8_formats
(video_url
, video_id
, ext
='mp4')
2485 entry_info_dict
['formats'] = self
._extract
_mpd
_formats
(video_url
, video_id
)
2487 entry_info_dict
['formats'] = self
._extract
_f
4m
_formats
(video_url
, video_id
)
2488 elif re
.search(r
'(?i)\.(?:ism|smil)/manifest', video_url
) and video_url
!= url
:
2489 # Just matching .ism/manifest is not enough to be reliably sure
2490 # whether it's actually an ISM manifest or some other streaming
2491 # manifest since there are various streaming URL formats
2492 # possible (see [1]) as well as some other shenanigans like
2493 # .smil/manifest URLs that actually serve an ISM (see [2]) and
2495 # Thus the most reasonable way to solve this is to delegate
2496 # to generic extractor in order to look into the contents of
2497 # the manifest itself.
2498 # 1. https://azure.microsoft.com/en-us/documentation/articles/media-services-deliver-content-overview/#streaming-url-formats
2499 # 2. https://svs.itworkscdn.net/lbcivod/smil:itwfcdn/lbci/170976.smil/Manifest
2500 entry_info_dict
= self
.url_result(
2501 smuggle_url(video_url
, {'to_generic': True}
),
2504 entry_info_dict
['url'] = video_url
2506 if entry_info_dict
.get('formats'):
2507 self
._sort
_formats
(entry_info_dict
['formats'])
2509 entries
.append(entry_info_dict
)
2511 if len(entries
) == 1:
2514 for num
, e
in enumerate(entries
, start
=1):
2515 # 'url' results don't have a title
2516 if e
.get('title') is not None:
2517 e
['title'] = '%s (%d)' % (e
['title'], num
)
2519 '_type': 'playlist',