]>
jfr.im git - yt-dlp.git/blob - youtube_dl/extractor/bbc.py
2 from __future__
import unicode_literals
5 import xml
. etree
. ElementTree
7 from . common
import InfoExtractor
16 from .. compat
import compat_HTTPError
19 class BBCCoUkIE ( InfoExtractor
):
21 IE_DESC
= 'BBC iPlayer'
22 _VALID_URL
= r
'https?://(?:www\.)?bbc\.co\.uk/(?:(?:(?:programmes|iplayer(?:/[^/]+)?/(?:episode|playlist))/)|music/clips[/#])(?P<id>[\da-z] {8} )'
24 _MEDIASELECTOR_URLS
= [
25 # Provides HQ HLS streams with even better quality that pc mediaset but fails
26 # with geolocation in some cases when it's even not geo restricted at all (e.g.
27 # http://www.bbc.co.uk/programmes/b06bp7lf)
28 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/iptv-all/vpid/ %s ' ,
29 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/ %s ' ,
32 _MEDIASELECTION_NS
= 'http://bbc.co.uk/2008/mp/mediaselection'
33 _EMP_PLAYLIST_NS
= 'http://bbc.co.uk/2008/emp/playlist'
42 'url' : 'http://www.bbc.co.uk/programmes/b039g8p7' ,
46 'title' : 'Kaleidoscope, Leonard Cohen' ,
47 'description' : 'The Canadian poet and songwriter reflects on his musical career.' ,
52 'skip_download' : True ,
56 'url' : 'http://www.bbc.co.uk/iplayer/episode/b00yng5w/The_Man_in_Black_Series_3_The_Printed_Name/' ,
60 'title' : 'The Man in Black: Series 3: The Printed Name' ,
61 'description' : "Mark Gatiss introduces Nicholas Pierpan's chilling tale of a writer's devilish pact with a mysterious man. Stars Ewan Bailey." ,
66 'skip_download' : True ,
68 'skip' : 'Episode is no longer available on BBC iPlayer Radio' ,
71 'url' : 'http://www.bbc.co.uk/iplayer/episode/b03vhd1f/The_Voice_UK_Series_3_Blind_Auditions_5/' ,
75 'title' : 'The Voice UK: Series 3: Blind Auditions 5' ,
76 'description' : "Emma Willis and Marvin Humes present the fifth set of blind auditions in the singing competition, as the coaches continue to build their teams based on voice alone." ,
81 'skip_download' : True ,
83 'skip' : 'Currently BBC iPlayer TV programmes are available to play in the UK only' ,
86 'url' : 'http://www.bbc.co.uk/iplayer/episode/p026c7jt/tomorrows-worlds-the-unearthly-history-of-science-fiction-2-invasion' ,
90 'title' : "Tomorrow's Worlds: The Unearthly History of Science Fiction" ,
91 'description' : '2. Invasion' ,
96 'skip_download' : True ,
98 'skip' : 'Currently BBC iPlayer TV programmes are available to play in the UK only' ,
100 'url' : 'http://www.bbc.co.uk/programmes/b04v20dw' ,
104 'title' : 'Pete Tong, The Essential New Tune Special' ,
105 'description' : "Pete has a very special mix - all of 2014's Essential New Tunes!" ,
110 'skip_download' : True ,
113 'url' : 'http://www.bbc.co.uk/music/clips/p02frcc3' ,
118 'title' : 'Pete Tong, Past, Present and Future Special, Madeon - After Hours mix' ,
119 'description' : 'French house superstar Madeon takes us out of the club and onto the after party.' ,
124 'skip_download' : True ,
127 'url' : 'http://www.bbc.co.uk/music/clips/p025c0zz' ,
132 'title' : 'Reading and Leeds Festival, 2014, Rae Morris - Closer (Live on BBC Three)' ,
133 'description' : 'Rae Morris performs Closer for BBC Three at Reading 2014' ,
138 'skip_download' : True ,
141 'url' : 'http://www.bbc.co.uk/iplayer/episode/b054fn09/ad/natural-world-20152016-2-super-powered-owls' ,
145 'title' : 'Natural World, 2015-2016: 2. Super Powered Owls' ,
146 'description' : 'md5:e4db5c937d0e95a7c6b5e654d429183d' ,
151 'skip_download' : True ,
153 'skip' : 'geolocation' ,
155 'url' : 'http://www.bbc.co.uk/iplayer/episode/b05zmgwn/royal-academy-summer-exhibition' ,
159 'description' : 'Kirsty Wark and Morgan Quaintance visit the Royal Academy as it prepares for its annual artistic extravaganza, meeting people who have come together to make the show unique.' ,
160 'title' : 'Royal Academy Summer Exhibition' ,
165 'skip_download' : True ,
167 'skip' : 'geolocation' ,
169 # iptv-all mediaset fails with geolocation however there is no geo restriction
170 # for this programme at all
171 'url' : 'http://www.bbc.co.uk/programmes/b06bp7lf' ,
175 'title' : "Annie Mac's Friday Night, B.Traits sits in for Annie" ,
176 'description' : 'B.Traits sits in for Annie Mac with a Mini-Mix from Disclosure.' ,
181 'skip_download' : True ,
184 'url' : 'http://www.bbc.co.uk/iplayer/playlist/p01dvks4' ,
185 'only_matching' : True ,
187 'url' : 'http://www.bbc.co.uk/music/clips#p02frcc3' ,
188 'only_matching' : True ,
190 'url' : 'http://www.bbc.co.uk/iplayer/cbeebies/episode/b0480276/bing-14-atchoo' ,
191 'only_matching' : True ,
195 class MediaSelectionError ( Exception ):
196 def __init__ ( self
, id ):
199 def _extract_asx_playlist ( self
, connection
, programme_id
):
200 asx
= self
._ download
_ xml
( connection
. get ( 'href' ), programme_id
, 'Downloading ASX playlist' )
201 return [ ref
. get ( 'href' ) for ref
in asx
. findall ( './Entry/ref' )]
203 def _extract_connection ( self
, connection
, programme_id
):
205 kind
= connection
. get ( 'kind' )
206 protocol
= connection
. get ( 'protocol' )
207 supplier
= connection
. get ( 'supplier' )
208 if protocol
== 'http' :
209 href
= connection
. get ( 'href' )
210 transfer_format
= connection
. get ( 'transferFormat' )
212 if supplier
== 'asx' :
213 for i
, ref
in enumerate ( self
._ extract
_ asx
_ playlist
( connection
, programme_id
)):
216 'format_id' : 'ref %s _ %s ' % ( i
, supplier
),
218 # Skip DASH until supported
219 elif transfer_format
== 'dash' :
221 elif transfer_format
== 'hls' :
222 m3u8_formats
= self
._ extract
_ m
3u8_ formats
(
223 href
, programme_id
, ext
= 'mp4' , entry_protocol
= 'm3u8_native' ,
224 m3u8_id
= supplier
, fatal
= False )
226 formats
. extend ( m3u8_formats
)
231 'format_id' : supplier
or kind
or protocol
,
233 elif protocol
== 'rtmp' :
234 application
= connection
. get ( 'application' , 'ondemand' )
235 auth_string
= connection
. get ( 'authString' )
236 identifier
= connection
. get ( 'identifier' )
237 server
= connection
. get ( 'server' )
239 'url' : ' %s :// %s / %s ? %s ' % ( protocol
, server
, application
, auth_string
),
240 'play_path' : identifier
,
241 'app' : ' %s ? %s ' % ( application
, auth_string
),
242 'page_url' : 'http://www.bbc.co.uk' ,
243 'player_url' : 'http://www.bbc.co.uk/emp/releases/iplayer/revisions/617463_618125_4/617463_618125_4_emp.swf' ,
246 'format_id' : supplier
,
250 def _extract_items ( self
, playlist
):
251 return playlist
. findall ( './ {%s} item' % self
._ EMP
_ PLAYLIST
_ NS
)
253 def _findall_ns ( self
, element
, xpath
):
255 for ns
in self
._ NAMESPACES
:
256 elements
. extend ( element
. findall ( xpath
% ns
))
259 def _extract_medias ( self
, media_selection
):
260 error
= media_selection
. find ( './ {%s} error' % self
._ MEDIASELECTION
_ NS
)
262 media_selection
. find ( './ {%s} error' % self
._ EMP
_ PLAYLIST
_ NS
)
263 if error
is not None :
264 raise BBCCoUkIE
. MediaSelectionError ( error
. get ( 'id' ))
265 return self
._ findall
_ ns
( media_selection
, './ {%s} media' )
267 def _extract_connections ( self
, media
):
268 return self
._ findall
_ ns
( media
, './ {%s} connection' )
270 def _extract_video ( self
, media
, programme_id
):
272 vbr
= int_or_none ( media
. get ( 'bitrate' ))
273 vcodec
= media
. get ( 'encoding' )
274 service
= media
. get ( 'service' )
275 width
= int_or_none ( media
. get ( 'width' ))
276 height
= int_or_none ( media
. get ( 'height' ))
277 file_size
= int_or_none ( media
. get ( 'media_file_size' ))
278 for connection
in self
._ extract
_ connections
( media
):
279 conn_formats
= self
._ extract
_ connection
( connection
, programme_id
)
280 for format
in conn_formats
:
286 'filesize' : file_size
,
289 format
[ 'format_id' ] = ' %s _ %s ' % ( service
, format
[ 'format_id' ])
290 formats
. extend ( conn_formats
)
293 def _extract_audio ( self
, media
, programme_id
):
295 abr
= int_or_none ( media
. get ( 'bitrate' ))
296 acodec
= media
. get ( 'encoding' )
297 service
= media
. get ( 'service' )
298 for connection
in self
._ extract
_ connections
( media
):
299 conn_formats
= self
._ extract
_ connection
( connection
, programme_id
)
300 for format
in conn_formats
:
302 'format_id' : ' %s _ %s ' % ( service
, format
[ 'format_id' ]),
306 formats
. extend ( conn_formats
)
309 def _get_subtitles ( self
, media
, programme_id
):
311 for connection
in self
._ extract
_ connections
( media
):
312 captions
= self
._ download
_ xml
( connection
. get ( 'href' ), programme_id
, 'Downloading captions' )
313 lang
= captions
. get ( ' {http://www.w3.org/XML/1998/namespace} lang' , 'en' )
316 'url' : connection
. get ( 'href' ),
322 def _raise_extractor_error ( self
, media_selection_error
):
323 raise ExtractorError (
324 ' %s returned error: %s ' % ( self
. IE_NAME
, media_selection_error
. id ),
327 def _download_media_selector ( self
, programme_id
):
328 last_exception
= None
329 for mediaselector_url
in self
._ MEDIASELECTOR
_U RLS
:
331 return self
._ download
_ media
_ selector
_u rl
(
332 mediaselector_url
% programme_id
, programme_id
)
333 except BBCCoUkIE
. MediaSelectionError
as e
:
334 if e
. id in ( 'notukerror' , 'geolocation' ):
337 self
._ raise
_ extractor
_ error
( e
)
338 self
._ raise
_ extractor
_ error
( last_exception
)
340 def _download_media_selector_url ( self
, url
, programme_id
= None ):
342 media_selection
= self
._ download
_ xml
(
343 url
, programme_id
, 'Downloading media selection XML' )
344 except ExtractorError
as ee
:
345 if isinstance ( ee
. cause
, compat_HTTPError
) and ee
. cause
. code
== 403 :
346 media_selection
= xml
. etree
. ElementTree
. fromstring ( ee
. cause
. read (). decode ( 'utf-8' ))
349 return self
._ process
_ media
_ selector
( media_selection
, programme_id
)
351 def _process_media_selector ( self
, media_selection
, programme_id
):
355 for media
in self
._ extract
_ medias
( media_selection
):
356 kind
= media
. get ( 'kind' )
358 formats
. extend ( self
._ extract
_ audio
( media
, programme_id
))
359 elif kind
== 'video' :
360 formats
. extend ( self
._ extract
_ video
( media
, programme_id
))
361 elif kind
== 'captions' :
362 subtitles
= self
. extract_subtitles ( media
, programme_id
)
363 return formats
, subtitles
365 def _download_playlist ( self
, playlist_id
):
367 playlist
= self
._ download
_ json
(
368 'http://www.bbc.co.uk/programmes/ %s /playlist.json' % playlist_id
,
369 playlist_id
, 'Downloading playlist JSON' )
371 version
= playlist
. get ( 'defaultAvailableVersion' )
373 smp_config
= version
[ 'smpConfig' ]
374 title
= smp_config
[ 'title' ]
375 description
= smp_config
[ 'summary' ]
376 for item
in smp_config
[ 'items' ]:
378 if kind
!= 'programme' and kind
!= 'radioProgramme' :
380 programme_id
= item
. get ( 'vpid' )
381 duration
= int_or_none ( item
. get ( 'duration' ))
382 formats
, subtitles
= self
._ download
_ media
_ selector
( programme_id
)
383 return programme_id
, title
, description
, duration
, formats
, subtitles
384 except ExtractorError
as ee
:
385 if not ( isinstance ( ee
. cause
, compat_HTTPError
) and ee
. cause
. code
== 404 ):
388 # fallback to legacy playlist
389 return self
._ process
_l egacy
_ playlist
( playlist_id
)
391 def _process_legacy_playlist_url ( self
, url
, display_id
):
392 playlist
= self
._ download
_l egacy
_ playlist
_u rl
( url
, display_id
)
393 return self
._ extract
_ from
_l egacy
_ playlist
( playlist
, display_id
)
395 def _process_legacy_playlist ( self
, playlist_id
):
396 return self
._ process
_l egacy
_ playlist
_u rl
(
397 'http://www.bbc.co.uk/iplayer/playlist/ %s ' % playlist_id
, playlist_id
)
399 def _download_legacy_playlist_url ( self
, url
, playlist_id
= None ):
400 return self
._ download
_ xml
(
401 url
, playlist_id
, 'Downloading legacy playlist XML' )
403 def _extract_from_legacy_playlist ( self
, playlist
, playlist_id
):
404 no_items
= playlist
. find ( './ {%s} noItems' % self
._ EMP
_ PLAYLIST
_ NS
)
405 if no_items
is not None :
406 reason
= no_items
. get ( 'reason' )
407 if reason
== 'preAvailability' :
408 msg
= 'Episode %s is not yet available' % playlist_id
409 elif reason
== 'postAvailability' :
410 msg
= 'Episode %s is no longer available' % playlist_id
411 elif reason
== 'noMedia' :
412 msg
= 'Episode %s is not currently available' % playlist_id
414 msg
= 'Episode %s is not available: %s ' % ( playlist_id
, reason
)
415 raise ExtractorError ( msg
, expected
= True )
417 for item
in self
._ extract
_ items
( playlist
):
418 kind
= item
. get ( 'kind' )
419 if kind
!= 'programme' and kind
!= 'radioProgramme' :
421 title
= playlist
. find ( './ {%s} title' % self
._ EMP
_ PLAYLIST
_ NS
). text
422 description_el
= playlist
. find ( './ {%s} summary' % self
._ EMP
_ PLAYLIST
_ NS
)
423 description
= description_el
. text
if description_el
else None
425 def get_programme_id ( item
):
426 def get_from_attributes ( item
):
427 for p
in ( 'identifier' , 'group' ):
429 if value
and re
. match ( r
'^[pb][\da-z] {7} $' , value
):
431 get_from_attributes ( item
)
432 mediator
= item
. find ( './ {%s} mediator' % self
._ EMP
_ PLAYLIST
_ NS
)
433 if mediator
is not None :
434 return get_from_attributes ( mediator
)
436 programme_id
= get_programme_id ( item
)
437 duration
= int_or_none ( item
. get ( 'duration' ))
440 formats
, subtitles
= self
._ download
_ media
_ selector
( programme_id
)
442 formats
, subtitles
= self
._ process
_ media
_ selector
( item
, playlist_id
)
443 programme_id
= playlist_id
445 return programme_id
, title
, description
, duration
, formats
, subtitles
447 def _real_extract ( self
, url
):
448 group_id
= self
._ match
_ id
( url
)
450 webpage
= self
._ download
_ webpage
( url
, group_id
, 'Downloading video page' )
454 tviplayer
= self
._ search
_ regex
(
455 r
'mediator\.bind\(( {.+?} )\s*,\s*document\.getElementById' ,
456 webpage
, 'player' , default
= None )
459 player
= self
._ parse
_ json
( tviplayer
, group_id
). get ( 'player' , {})
460 duration
= int_or_none ( player
. get ( 'duration' ))
461 programme_id
= player
. get ( 'vpid' )
464 programme_id
= self
._ search
_ regex
(
465 r
'"vpid"\s*:\s*"([\da-z] {8} )"' , webpage
, 'vpid' , fatal
= False , default
= None )
468 formats
, subtitles
= self
._ download
_ media
_ selector
( programme_id
)
469 title
= self
._ og
_ search
_ title
( webpage
)
470 description
= self
._ search
_ regex
(
471 r
'<p class="[^"]*medium-description[^"]*">([^<]+)</p>' ,
472 webpage
, 'description' , fatal
= False )
474 programme_id
, title
, description
, duration
, formats
, subtitles
= self
._ download
_ playlist
( group_id
)
476 self
._ sort
_ formats
( formats
)
481 'description' : description
,
482 'thumbnail' : self
._ og
_ search
_ thumbnail
( webpage
, default
= None ),
483 'duration' : duration
,
485 'subtitles' : subtitles
,
489 class BBCIE ( BBCCoUkIE
):
492 _VALID_URL
= r
'https?://(?:www\.)?bbc\.(?:com|co\.uk)/(?:[^/]+/)+(?P<id>[^/#?]+)'
494 _MEDIASELECTOR_URLS
= [
495 # Provides more formats, namely direct mp4 links, but fails on some videos with
496 # notukerror for non UK (?) users (e.g.
497 # http://www.bbc.com/travel/story/20150625-sri-lankas-spicy-secret)
498 'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/ %s ' ,
499 # Provides fewer formats, but works everywhere for everybody (hopefully)
500 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/journalism-pc/vpid/ %s ' ,
504 # article with multiple videos embedded with data-media-meta containing
505 # playlist.sxml, externalId and no direct video links
506 'url' : 'http://www.bbc.com/news/world-europe-32668511' ,
508 'id' : 'world-europe-32668511' ,
509 'title' : 'Russia stages massive WW2 parade despite Western boycott' ,
510 'description' : 'md5:00ff61976f6081841f759a08bf78cc9c' ,
514 # article with multiple videos embedded with data-media-meta (more videos)
515 'url' : 'http://www.bbc.com/news/business-28299555' ,
517 'id' : 'business-28299555' ,
518 'title' : 'Farnborough Airshow: Video highlights' ,
519 'description' : 'BBC reports and video highlights at the Farnborough Airshow.' ,
524 # article with multiple videos embedded with `new SMP()`
525 'url' : 'http://www.bbc.co.uk/blogs/adamcurtis/entries/3662a707-0af9-3149-963f-47bea720b460' ,
527 'id' : '3662a707-0af9-3149-963f-47bea720b460' ,
528 'title' : 'BBC Blogs - Adam Curtis - BUGGER' ,
530 'playlist_count' : 18 ,
532 # single video embedded with mediaAssetPage.init()
533 'url' : 'http://www.bbc.com/news/world-europe-32041533' ,
537 'title' : 'Aerial footage showed the site of the crash in the Alps - courtesy BFM TV' ,
539 'timestamp' : 1427219242 ,
540 'upload_date' : '20150324' ,
544 'skip_download' : True ,
547 # article with single video embedded with data-media-meta containing
548 # direct video links (for now these are extracted) and playlist.xml (with
549 # media items as f4m and m3u8 - currently unsupported)
550 'url' : 'http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu' ,
552 'id' : '150615_telabyad_kentin_cogu' ,
554 'title' : "YPG: Tel Abyad'ın tamamı kontrolümüzde" ,
556 'timestamp' : 1434397334 ,
557 'upload_date' : '20150615' ,
560 'skip_download' : True ,
563 # single video embedded with playlist.sxml in data-playable
564 'url' : 'http://www.bbc.com/turkce/multimedya/2015/10/151010_vid_ankara_patlama_ani' ,
566 'id' : '151010_vid_ankara_patlama_ani' ,
568 'title' : "Ankara'da patlama anı" ,
569 'timestamp' : 1444480325 ,
570 'upload_date' : '20151010' ,
573 'skip_download' : True ,
576 # single video embedded with mediaAssetPage.init() (regional section)
577 'url' : 'http://www.bbc.com/mundo/video_fotos/2015/06/150619_video_honduras_militares_hospitales_corrupcion_aw' ,
579 'id' : '150619_video_honduras_militares_hospitales_corrupcion_aw' ,
581 'title' : 'Honduras militariza sus hospitales por nuevo escándalo de corrupción' ,
583 'timestamp' : 1434713142 ,
584 'upload_date' : '20150619' ,
587 'skip_download' : True ,
590 # single video from video playlist embedded with vxp-playlist-data JSON
591 'url' : 'http://www.bbc.com/news/video_and_audio/must_see/33376376' ,
595 'title' : '''Judge Mindy Glazer: "I'm sorry to see you here... I always wondered what happened to you"''' ,
599 'skip_download' : True ,
602 # single video story with digitalData
603 'url' : 'http://www.bbc.com/travel/story/20150625-sri-lankas-spicy-secret' ,
607 'title' : 'Sri Lanka’s spicy secret' ,
608 'description' : 'As a new train line to Jaffna opens up the country’s north, travellers can experience a truly distinct slice of Tamil culture.' ,
609 'timestamp' : 1437674293 ,
610 'upload_date' : '20150723' ,
614 'skip_download' : True ,
617 # single video story without digitalData
618 'url' : 'http://www.bbc.com/autos/story/20130513-hyundais-rock-star' ,
622 'title' : 'Hyundai Santa Fe Sport: Rock star' ,
623 'description' : 'md5:b042a26142c4154a6e472933cf20793d' ,
624 'timestamp' : 1368473503 ,
625 'upload_date' : '20130513' ,
629 'skip_download' : True ,
632 # single video with playlist.sxml URL
633 'url' : 'http://www.bbc.com/sport/0/football/33653409' ,
637 'title' : 'Transfers: Cristiano Ronaldo to Man Utd, Arsenal to spend?' ,
638 'description' : 'md5:398fca0e2e701c609d726e034fa1fc89' ,
643 'skip_download' : True ,
646 # article with multiple videos embedded with playlist.sxml
647 'url' : 'http://www.bbc.com/sport/0/football/34475836' ,
650 'title' : 'What Liverpool can expect from Klopp' ,
654 # single video with playlist URL from weather section
655 'url' : 'http://www.bbc.com/weather/features/33601775' ,
656 'only_matching' : True ,
658 # custom redirection to www.bbc.com
659 'url' : 'http://www.bbc.co.uk/news/science-environment-33661876' ,
660 'only_matching' : True ,
664 def suitable ( cls
, url
):
665 return False if BBCCoUkIE
. suitable ( url
) else super ( BBCIE
, cls
). suitable ( url
)
667 def _extract_from_media_meta ( self
, media_meta
, video_id
):
668 # Direct links to media in media metadata (e.g.
669 # http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu)
670 # TODO: there are also f4m and m3u8 streams incorporated in playlist.sxml
671 source_files
= media_meta
. get ( 'sourceFiles' )
675 'format_id' : format_id
,
676 'ext' : f
. get ( 'encoding' ),
677 'tbr' : float_or_none ( f
. get ( 'bitrate' ), 1000 ),
678 'filesize' : int_or_none ( f
. get ( 'filesize' )),
679 } for format_id
, f
in source_files
. items () if f
. get ( 'url' )], []
681 programme_id
= media_meta
. get ( 'externalId' )
683 return self
._ download
_ media
_ selector
( programme_id
)
685 # Process playlist.sxml as legacy playlist
686 href
= media_meta
. get ( 'href' )
688 playlist
= self
._ download
_l egacy
_ playlist
_u rl
( href
)
689 _
, _
, _
, _
, formats
, subtitles
= self
._ extract
_ from
_l egacy
_ playlist
( playlist
, video_id
)
690 return formats
, subtitles
694 def _extract_from_playlist_sxml ( self
, url
, playlist_id
, timestamp
):
695 programme_id
, title
, description
, duration
, formats
, subtitles
= \
696 self
._ process
_l egacy
_ playlist
_u rl
( url
, playlist_id
)
697 self
._ sort
_ formats
( formats
)
701 'description' : description
,
702 'duration' : duration
,
703 'timestamp' : timestamp
,
705 'subtitles' : subtitles
,
708 def _real_extract ( self
, url
):
709 playlist_id
= self
._ match
_ id
( url
)
711 webpage
= self
._ download
_ webpage
( url
, playlist_id
)
713 timestamp
= parse_iso8601 ( self
._ search
_ regex
(
714 [ r
'"datePublished":\s*"([^"]+)' ,
715 r
'<meta[^>]+property="article:published_time"[^>]+content="([^"]+)"' ,
716 r
'itemprop="datePublished"[^>]+datetime="([^"]+)"' ],
717 webpage
, 'date' , default
= None ))
719 # article with multiple videos embedded with playlist.sxml (e.g.
720 # http://www.bbc.com/sport/0/football/34475836)
721 playlists
= re
. findall ( r
'<param[^>]+name="playlist"[^>]+value="([^"]+)"' , webpage
)
724 self
._ extract
_ from
_ playlist
_ sxml
( playlist_url
, playlist_id
, timestamp
)
725 for playlist_url
in playlists
]
726 playlist_title
= self
._ og
_ search
_ title
( webpage
)
727 playlist_description
= self
._ og
_ search
_ description
( webpage
, default
= None )
728 return self
. playlist_result ( entries
, playlist_id
, playlist_title
, playlist_description
)
730 # single video story (e.g. http://www.bbc.com/travel/story/20150625-sri-lankas-spicy-secret)
731 programme_id
= self
._ search
_ regex
(
732 [ r
'data-video-player-vpid="([\da-z] {8} )"' ,
733 r
'<param[^>]+name="externalIdentifier"[^>]+value="([\da-z] {8} )"' ],
734 webpage
, 'vpid' , default
= None )
738 # single video in news article embedded with data-playable (e.g.
739 # http://www.bbc.com/news/world-us-canada-34473351)
740 data_playable
= self
._ parse
_ json
(
741 unescapeHTML ( self
._ search
_ regex
(
742 r
'data-playable="( {.+?} )"' , webpage
, 'data playable' , default
= '{}' )),
743 programme_id
, fatal
= False )
745 # data-playable has video vpid in settings.playlistObject.items (e.g.
746 # http://www.bbc.com/news/world-us-canada-34473351)
747 settings
= data_playable
. get ( 'settings' , {})
749 playlist_object
= settings
. get ( 'playlistObject' , {})
751 items
= playlist_object
. get ( 'items' )
752 if items
and isinstance ( items
, list ):
753 duration
= int_or_none ( items
[ 0 ]. get ( 'duration' ))
754 programme_id
= items
[ 0 ]. get ( 'vpid' )
756 # data-playable has no vpid but has a playlist.sxml URLs
757 # in otherSettings.playlist (e.g.
758 # http://www.bbc.com/turkce/multimedya/2015/10/151010_vid_ankara_patlama_ani)
759 playlist
= data_playable
. get ( 'otherSettings' , {}).get('playlist', {}
)
761 return self
._ extract
_ from
_ playlist
_ sxml
(
762 playlist
. get ( 'progressiveDownloadUrl' ), playlist_id
, timestamp
)
765 formats
, subtitles
= self
._ download
_ media
_ selector
( programme_id
)
766 self
._ sort
_ formats
( formats
)
767 # digitalData may be missing (e.g. http://www.bbc.com/autos/story/20130513-hyundais-rock-star)
768 digital_data
= self
._ parse
_ json
(
770 r
'var\s+digitalData\s*=\s*( {.+?} );?\n' , webpage
, 'digital data' , default
= '{}' ),
771 programme_id
, fatal
= False )
772 page_info
= digital_data
. get ( 'page' , {}).get('pageInfo', {}
)
773 title
= page_info
. get ( 'pageName' ) or self
._ og
_ search
_ title
( webpage
)
774 description
= page_info
. get ( 'description' ) or self
._ og
_ search
_ description
( webpage
)
775 timestamp
= parse_iso8601 ( page_info
. get ( 'publicationDate' )) or timestamp
779 'description' : description
,
780 'timestamp' : timestamp
,
781 'duration' : duration
,
783 'subtitles' : subtitles
,
786 playlist_title
= self
._ html
_ search
_ regex
(
787 r
'<title>(.*?)(?:\s*-\s*BBC [^ ]+)?</title>' , webpage
, 'playlist title' )
788 playlist_description
= self
._ og
_ search
_ description
( webpage
, default
= None )
790 def extract_all ( pattern
):
791 return list ( filter ( None , map (
792 lambda s
: self
._ parse
_ json
( s
, playlist_id
, fatal
= False ),
793 re
. findall ( pattern
, webpage
))))
795 # Multiple video article (e.g.
796 # http://www.bbc.co.uk/blogs/adamcurtis/entries/3662a707-0af9-3149-963f-47bea720b460)
797 EMBED_URL
= r
'https?://(?:www\.)?bbc\.co\.uk/(?:[^/]+/)+[\da-z] {8} (?:\b[^"]+)?'
799 for match
in extract_all ( r
'new\s+SMP\(( {.+?} )\)' ):
800 embed_url
= match
. get ( 'playerSettings' , {}). get ( 'externalEmbedUrl' )
801 if embed_url
and re
. match ( EMBED_URL
, embed_url
):
802 entries
. append ( embed_url
)
803 entries
. extend ( re
. findall (
804 r
'setPlaylist\("( %s )"\)' % EMBED_URL
, webpage
))
806 return self
. playlist_result (
807 [ self
. url_result ( entry
, 'BBCCoUk' ) for entry
in entries
],
808 playlist_id
, playlist_title
, playlist_description
)
810 # Multiple video article (e.g. http://www.bbc.com/news/world-europe-32668511)
811 medias
= extract_all ( r
"data-media-meta='( {[^']+} )'" )
814 # Single video article (e.g. http://www.bbc.com/news/video_and_audio/international)
815 media_asset
= self
._ search
_ regex
(
816 r
'mediaAssetPage\.init\(\s*( {.+?} ), "/' ,
817 webpage
, 'media asset' , default
= None )
819 media_asset_page
= self
._ parse
_ json
( media_asset
, playlist_id
, fatal
= False )
821 for video
in media_asset_page
. get ( 'videos' , {}). values ():
822 medias
. extend ( video
. values ())
825 # Multiple video playlist with single `now playing` entry (e.g.
826 # http://www.bbc.com/news/video_and_audio/must_see/33767813)
827 vxp_playlist
= self
._ parse
_ json
(
829 r
'<script[^>]+class="vxp-playlist-data"[^>]+type="application/json"[^>]*>([^<]+)</script>' ,
830 webpage
, 'playlist data' ),
833 for item
in vxp_playlist
:
834 media
= item
. get ( 'media' )
837 playlist_medias
. append ( media
)
838 # Download single video if found media with asset id matching the video id from URL
839 if item
. get ( 'advert' , {}). get ( 'assetId' ) == playlist_id
:
842 # Fallback to the whole playlist
844 medias
= playlist_medias
847 for num
, media_meta
in enumerate ( medias
, start
= 1 ):
848 formats
, subtitles
= self
._ extract
_ from
_ media
_ meta
( media_meta
, playlist_id
)
851 self
._ sort
_ formats
( formats
)
853 video_id
= media_meta
. get ( 'externalId' )
855 video_id
= playlist_id
if len ( medias
) == 1 else ' %s-%s ' % ( playlist_id
, num
)
857 title
= media_meta
. get ( 'caption' )
859 title
= playlist_title
if len ( medias
) == 1 else ' %s - Video %s ' % ( playlist_title
, num
)
861 duration
= int_or_none ( media_meta
. get ( 'durationInSeconds' )) or parse_duration ( media_meta
. get ( 'duration' ))
864 for image
in media_meta
. get ( 'images' , {}). values ():
865 images
. extend ( image
. values ())
866 if 'image' in media_meta
:
867 images
. append ( media_meta
[ 'image' ])
870 'url' : image
. get ( 'href' ),
871 'width' : int_or_none ( image
. get ( 'width' )),
872 'height' : int_or_none ( image
. get ( 'height' )),
873 } for image
in images
]
878 'thumbnails' : thumbnails
,
879 'duration' : duration
,
880 'timestamp' : timestamp
,
882 'subtitles' : subtitles
,
885 return self
. playlist_result ( entries
, playlist_id
, playlist_title
, playlist_description
)