]>
jfr.im git - yt-dlp.git/blob - youtube_dl/extractor/bbc.py
2 from __future__
import unicode_literals
5 import xml
. etree
. ElementTree
7 from . common
import InfoExtractor
16 from .. compat
import compat_HTTPError
19 class BBCCoUkIE ( InfoExtractor
):
21 IE_DESC
= 'BBC iPlayer'
22 _VALID_URL
= r
'https?://(?:www\.)?bbc\.co\.uk/(?:(?:(?:programmes|iplayer(?:/[^/]+)?/(?:episode|playlist))/)|music/clips[/#])(?P<id>[\da-z] {8} )'
24 _MEDIASELECTOR_URLS
= [
25 # Provides HQ HLS streams with even better quality that pc mediaset but fails
26 # with geolocation in some cases when it's even not geo restricted at all (e.g.
27 # http://www.bbc.co.uk/programmes/b06bp7lf)
28 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/iptv-all/vpid/ %s ' ,
29 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/ %s ' ,
34 'url' : 'http://www.bbc.co.uk/programmes/b039g8p7' ,
38 'title' : 'Kaleidoscope, Leonard Cohen' ,
39 'description' : 'The Canadian poet and songwriter reflects on his musical career.' ,
44 'skip_download' : True ,
48 'url' : 'http://www.bbc.co.uk/iplayer/episode/b00yng5w/The_Man_in_Black_Series_3_The_Printed_Name/' ,
52 'title' : 'The Man in Black: Series 3: The Printed Name' ,
53 'description' : "Mark Gatiss introduces Nicholas Pierpan's chilling tale of a writer's devilish pact with a mysterious man. Stars Ewan Bailey." ,
58 'skip_download' : True ,
60 'skip' : 'Episode is no longer available on BBC iPlayer Radio' ,
63 'url' : 'http://www.bbc.co.uk/iplayer/episode/b03vhd1f/The_Voice_UK_Series_3_Blind_Auditions_5/' ,
67 'title' : 'The Voice UK: Series 3: Blind Auditions 5' ,
68 'description' : "Emma Willis and Marvin Humes present the fifth set of blind auditions in the singing competition, as the coaches continue to build their teams based on voice alone." ,
73 'skip_download' : True ,
75 'skip' : 'Currently BBC iPlayer TV programmes are available to play in the UK only' ,
78 'url' : 'http://www.bbc.co.uk/iplayer/episode/p026c7jt/tomorrows-worlds-the-unearthly-history-of-science-fiction-2-invasion' ,
82 'title' : "Tomorrow's Worlds: The Unearthly History of Science Fiction" ,
83 'description' : '2. Invasion' ,
88 'skip_download' : True ,
90 'skip' : 'Currently BBC iPlayer TV programmes are available to play in the UK only' ,
92 'url' : 'http://www.bbc.co.uk/programmes/b04v20dw' ,
96 'title' : 'Pete Tong, The Essential New Tune Special' ,
97 'description' : "Pete has a very special mix - all of 2014's Essential New Tunes!" ,
102 'skip_download' : True ,
105 'url' : 'http://www.bbc.co.uk/music/clips/p02frcc3' ,
110 'title' : 'Pete Tong, Past, Present and Future Special, Madeon - After Hours mix' ,
111 'description' : 'French house superstar Madeon takes us out of the club and onto the after party.' ,
116 'skip_download' : True ,
119 'url' : 'http://www.bbc.co.uk/music/clips/p025c0zz' ,
124 'title' : 'Reading and Leeds Festival, 2014, Rae Morris - Closer (Live on BBC Three)' ,
125 'description' : 'Rae Morris performs Closer for BBC Three at Reading 2014' ,
130 'skip_download' : True ,
133 'url' : 'http://www.bbc.co.uk/iplayer/episode/b054fn09/ad/natural-world-20152016-2-super-powered-owls' ,
137 'title' : 'Natural World, 2015-2016: 2. Super Powered Owls' ,
138 'description' : 'md5:e4db5c937d0e95a7c6b5e654d429183d' ,
143 'skip_download' : True ,
145 'skip' : 'geolocation' ,
147 'url' : 'http://www.bbc.co.uk/iplayer/episode/b05zmgwn/royal-academy-summer-exhibition' ,
151 'description' : 'Kirsty Wark and Morgan Quaintance visit the Royal Academy as it prepares for its annual artistic extravaganza, meeting people who have come together to make the show unique.' ,
152 'title' : 'Royal Academy Summer Exhibition' ,
157 'skip_download' : True ,
159 'skip' : 'geolocation' ,
161 # iptv-all mediaset fails with geolocation however there is no geo restriction
162 # for this programme at all
163 'url' : 'http://www.bbc.co.uk/programmes/b06bp7lf' ,
167 'title' : "Annie Mac's Friday Night, B.Traits sits in for Annie" ,
168 'description' : 'B.Traits sits in for Annie Mac with a Mini-Mix from Disclosure.' ,
173 'skip_download' : True ,
176 'url' : 'http://www.bbc.co.uk/iplayer/playlist/p01dvks4' ,
177 'only_matching' : True ,
179 'url' : 'http://www.bbc.co.uk/music/clips#p02frcc3' ,
180 'only_matching' : True ,
182 'url' : 'http://www.bbc.co.uk/iplayer/cbeebies/episode/b0480276/bing-14-atchoo' ,
183 'only_matching' : True ,
187 class MediaSelectionError ( Exception ):
188 def __init__ ( self
, id ):
191 def _extract_asx_playlist ( self
, connection
, programme_id
):
192 asx
= self
._ download
_ xml
( connection
. get ( 'href' ), programme_id
, 'Downloading ASX playlist' )
193 return [ ref
. get ( 'href' ) for ref
in asx
. findall ( './Entry/ref' )]
195 def _extract_connection ( self
, connection
, programme_id
):
197 protocol
= connection
. get ( 'protocol' )
198 supplier
= connection
. get ( 'supplier' )
199 if protocol
== 'http' :
200 href
= connection
. get ( 'href' )
201 transfer_format
= connection
. get ( 'transferFormat' )
203 if supplier
== 'asx' :
204 for i
, ref
in enumerate ( self
._ extract
_ asx
_ playlist
( connection
, programme_id
)):
207 'format_id' : 'ref %s _ %s ' % ( i
, supplier
),
209 # Skip DASH until supported
210 elif transfer_format
== 'dash' :
212 elif transfer_format
== 'hls' :
213 m3u8_formats
= self
._ extract
_ m
3u8_ formats
(
214 href
, programme_id
, ext
= 'mp4' , entry_protocol
= 'm3u8_native' ,
215 m3u8_id
= supplier
, fatal
= False )
217 formats
. extend ( m3u8_formats
)
222 'format_id' : supplier
,
224 elif protocol
== 'rtmp' :
225 application
= connection
. get ( 'application' , 'ondemand' )
226 auth_string
= connection
. get ( 'authString' )
227 identifier
= connection
. get ( 'identifier' )
228 server
= connection
. get ( 'server' )
230 'url' : ' %s :// %s / %s ? %s ' % ( protocol
, server
, application
, auth_string
),
231 'play_path' : identifier
,
232 'app' : ' %s ? %s ' % ( application
, auth_string
),
233 'page_url' : 'http://www.bbc.co.uk' ,
234 'player_url' : 'http://www.bbc.co.uk/emp/releases/iplayer/revisions/617463_618125_4/617463_618125_4_emp.swf' ,
237 'format_id' : supplier
,
241 def _extract_items ( self
, playlist
):
242 return playlist
. findall ( './ {http://bbc.co.uk/2008/emp/playlist} item' )
244 def _extract_medias ( self
, media_selection
):
245 error
= media_selection
. find ( './ {http://bbc.co.uk/2008/mp/mediaselection} error' )
246 if error
is not None :
247 raise BBCCoUkIE
. MediaSelectionError ( error
. get ( 'id' ))
248 return media_selection
. findall ( './ {http://bbc.co.uk/2008/mp/mediaselection} media' )
250 def _extract_connections ( self
, media
):
251 return media
. findall ( './ {http://bbc.co.uk/2008/mp/mediaselection} connection' )
253 def _extract_video ( self
, media
, programme_id
):
255 vbr
= int_or_none ( media
. get ( 'bitrate' ))
256 vcodec
= media
. get ( 'encoding' )
257 service
= media
. get ( 'service' )
258 width
= int_or_none ( media
. get ( 'width' ))
259 height
= int_or_none ( media
. get ( 'height' ))
260 file_size
= int_or_none ( media
. get ( 'media_file_size' ))
261 for connection
in self
._ extract
_ connections
( media
):
262 conn_formats
= self
._ extract
_ connection
( connection
, programme_id
)
263 for format
in conn_formats
:
265 'format_id' : ' %s _ %s ' % ( service
, format
[ 'format_id' ]),
270 'filesize' : file_size
,
272 formats
. extend ( conn_formats
)
275 def _extract_audio ( self
, media
, programme_id
):
277 abr
= int_or_none ( media
. get ( 'bitrate' ))
278 acodec
= media
. get ( 'encoding' )
279 service
= media
. get ( 'service' )
280 for connection
in self
._ extract
_ connections
( media
):
281 conn_formats
= self
._ extract
_ connection
( connection
, programme_id
)
282 for format
in conn_formats
:
284 'format_id' : ' %s _ %s ' % ( service
, format
[ 'format_id' ]),
288 formats
. extend ( conn_formats
)
291 def _get_subtitles ( self
, media
, programme_id
):
293 for connection
in self
._ extract
_ connections
( media
):
294 captions
= self
._ download
_ xml
( connection
. get ( 'href' ), programme_id
, 'Downloading captions' )
295 lang
= captions
. get ( ' {http://www.w3.org/XML/1998/namespace} lang' , 'en' )
298 'url' : connection
. get ( 'href' ),
304 def _raise_extractor_error ( self
, media_selection_error
):
305 raise ExtractorError (
306 ' %s returned error: %s ' % ( self
. IE_NAME
, media_selection_error
. id ),
309 def _download_media_selector ( self
, programme_id
):
310 last_exception
= None
311 for mediaselector_url
in self
._ MEDIASELECTOR
_U RLS
:
313 return self
._ download
_ media
_ selector
_u rl
(
314 mediaselector_url
% programme_id
, programme_id
)
315 except BBCCoUkIE
. MediaSelectionError
as e
:
316 if e
. id in ( 'notukerror' , 'geolocation' ):
319 self
._ raise
_ extractor
_ error
( e
)
320 self
._ raise
_ extractor
_ error
( last_exception
)
322 def _download_media_selector_url ( self
, url
, programme_id
= None ):
324 media_selection
= self
._ download
_ xml
(
325 url
, programme_id
, 'Downloading media selection XML' )
326 except ExtractorError
as ee
:
327 if isinstance ( ee
. cause
, compat_HTTPError
) and ee
. cause
. code
== 403 :
328 media_selection
= xml
. etree
. ElementTree
. fromstring ( ee
. cause
. read (). decode ( 'utf-8' ))
331 return self
._ process
_ media
_ selector
( media_selection
, programme_id
)
333 def _process_media_selector ( self
, media_selection
, programme_id
):
337 for media
in self
._ extract
_ medias
( media_selection
):
338 kind
= media
. get ( 'kind' )
340 formats
. extend ( self
._ extract
_ audio
( media
, programme_id
))
341 elif kind
== 'video' :
342 formats
. extend ( self
._ extract
_ video
( media
, programme_id
))
343 elif kind
== 'captions' :
344 subtitles
= self
. extract_subtitles ( media
, programme_id
)
345 return formats
, subtitles
347 def _download_playlist ( self
, playlist_id
):
349 playlist
= self
._ download
_ json
(
350 'http://www.bbc.co.uk/programmes/ %s /playlist.json' % playlist_id
,
351 playlist_id
, 'Downloading playlist JSON' )
353 version
= playlist
. get ( 'defaultAvailableVersion' )
355 smp_config
= version
[ 'smpConfig' ]
356 title
= smp_config
[ 'title' ]
357 description
= smp_config
[ 'summary' ]
358 for item
in smp_config
[ 'items' ]:
360 if kind
!= 'programme' and kind
!= 'radioProgramme' :
362 programme_id
= item
. get ( 'vpid' )
363 duration
= int_or_none ( item
. get ( 'duration' ))
364 formats
, subtitles
= self
._ download
_ media
_ selector
( programme_id
)
365 return programme_id
, title
, description
, duration
, formats
, subtitles
366 except ExtractorError
as ee
:
367 if not ( isinstance ( ee
. cause
, compat_HTTPError
) and ee
. cause
. code
== 404 ):
370 # fallback to legacy playlist
371 return self
._ process
_l egacy
_ playlist
( playlist_id
)
373 def _process_legacy_playlist_url ( self
, url
, display_id
):
374 playlist
= self
._ download
_l egacy
_ playlist
_u rl
( url
, display_id
)
375 return self
._ extract
_ from
_l egacy
_ playlist
( playlist
, display_id
)
377 def _process_legacy_playlist ( self
, playlist_id
):
378 return self
._ process
_l egacy
_ playlist
_u rl
(
379 'http://www.bbc.co.uk/iplayer/playlist/ %s ' % playlist_id
, playlist_id
)
381 def _download_legacy_playlist_url ( self
, url
, playlist_id
= None ):
382 return self
._ download
_ xml
(
383 url
, playlist_id
, 'Downloading legacy playlist XML' )
385 def _extract_from_legacy_playlist ( self
, playlist
, playlist_id
):
386 no_items
= playlist
. find ( './ {http://bbc.co.uk/2008/emp/playlist} noItems' )
387 if no_items
is not None :
388 reason
= no_items
. get ( 'reason' )
389 if reason
== 'preAvailability' :
390 msg
= 'Episode %s is not yet available' % playlist_id
391 elif reason
== 'postAvailability' :
392 msg
= 'Episode %s is no longer available' % playlist_id
393 elif reason
== 'noMedia' :
394 msg
= 'Episode %s is not currently available' % playlist_id
396 msg
= 'Episode %s is not available: %s ' % ( playlist_id
, reason
)
397 raise ExtractorError ( msg
, expected
= True )
399 for item
in self
._ extract
_ items
( playlist
):
400 kind
= item
. get ( 'kind' )
401 if kind
!= 'programme' and kind
!= 'radioProgramme' :
403 title
= playlist
. find ( './ {http://bbc.co.uk/2008/emp/playlist} title' ). text
404 description_el
= playlist
. find ( './ {http://bbc.co.uk/2008/emp/playlist} summary' )
405 description
= description_el
. text
if description_el
else None
407 def get_programme_id ( item
):
408 def get_from_attributes ( item
):
409 for p
in ( 'identifier' , 'group' ):
411 if value
and re
. match ( r
'^[pb][\da-z] {7} $' , value
):
413 get_from_attributes ( item
)
414 mediator
= item
. find ( './ {http://bbc.co.uk/2008/emp/playlist} mediator' )
415 if mediator
is not None :
416 return get_from_attributes ( mediator
)
418 programme_id
= get_programme_id ( item
)
419 duration
= int_or_none ( item
. get ( 'duration' ))
420 # TODO: programme_id can be None and media items can be incorporated right inside
421 # playlist's item (e.g. http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu)
423 formats
, subtitles
= self
._ download
_ media
_ selector
( programme_id
)
425 return programme_id
, title
, description
, duration
, formats
, subtitles
427 def _real_extract ( self
, url
):
428 group_id
= self
._ match
_ id
( url
)
430 webpage
= self
._ download
_ webpage
( url
, group_id
, 'Downloading video page' )
434 tviplayer
= self
._ search
_ regex
(
435 r
'mediator\.bind\(( {.+?} )\s*,\s*document\.getElementById' ,
436 webpage
, 'player' , default
= None )
439 player
= self
._ parse
_ json
( tviplayer
, group_id
). get ( 'player' , {})
440 duration
= int_or_none ( player
. get ( 'duration' ))
441 programme_id
= player
. get ( 'vpid' )
444 programme_id
= self
._ search
_ regex
(
445 r
'"vpid"\s*:\s*"([\da-z] {8} )"' , webpage
, 'vpid' , fatal
= False , default
= None )
448 formats
, subtitles
= self
._ download
_ media
_ selector
( programme_id
)
449 title
= self
._ og
_ search
_ title
( webpage
)
450 description
= self
._ search
_ regex
(
451 r
'<p class="[^"]*medium-description[^"]*">([^<]+)</p>' ,
452 webpage
, 'description' , fatal
= False )
454 programme_id
, title
, description
, duration
, formats
, subtitles
= self
._ download
_ playlist
( group_id
)
456 self
._ sort
_ formats
( formats
)
461 'description' : description
,
462 'thumbnail' : self
._ og
_ search
_ thumbnail
( webpage
, default
= None ),
463 'duration' : duration
,
465 'subtitles' : subtitles
,
469 class BBCIE ( BBCCoUkIE
):
472 _VALID_URL
= r
'https?://(?:www\.)?bbc\.(?:com|co\.uk)/(?:[^/]+/)+(?P<id>[^/#?]+)'
474 _MEDIASELECTOR_URLS
= [
475 # Provides more formats, namely direct mp4 links, but fails on some videos with
476 # notukerror for non UK (?) users (e.g.
477 # http://www.bbc.com/travel/story/20150625-sri-lankas-spicy-secret)
478 'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/ %s ' ,
479 # Provides fewer formats, but works everywhere for everybody (hopefully)
480 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/journalism-pc/vpid/ %s ' ,
484 # article with multiple videos embedded with data-media-meta containing
485 # playlist.sxml, externalId and no direct video links
486 'url' : 'http://www.bbc.com/news/world-europe-32668511' ,
488 'id' : 'world-europe-32668511' ,
489 'title' : 'Russia stages massive WW2 parade despite Western boycott' ,
490 'description' : 'md5:00ff61976f6081841f759a08bf78cc9c' ,
494 # article with multiple videos embedded with data-media-meta (more videos)
495 'url' : 'http://www.bbc.com/news/business-28299555' ,
497 'id' : 'business-28299555' ,
498 'title' : 'Farnborough Airshow: Video highlights' ,
499 'description' : 'BBC reports and video highlights at the Farnborough Airshow.' ,
504 # article with multiple videos embedded with `new SMP()`
505 'url' : 'http://www.bbc.co.uk/blogs/adamcurtis/entries/3662a707-0af9-3149-963f-47bea720b460' ,
507 'id' : '3662a707-0af9-3149-963f-47bea720b460' ,
508 'title' : 'BBC Blogs - Adam Curtis - BUGGER' ,
510 'playlist_count' : 18 ,
512 # single video embedded with mediaAssetPage.init()
513 'url' : 'http://www.bbc.com/news/world-europe-32041533' ,
517 'title' : 'Aerial footage showed the site of the crash in the Alps - courtesy BFM TV' ,
519 'timestamp' : 1427219242 ,
520 'upload_date' : '20150324' ,
524 'skip_download' : True ,
527 # article with single video embedded with data-media-meta containing
528 # direct video links (for now these are extracted) and playlist.xml (with
529 # media items as f4m and m3u8 - currently unsupported)
530 'url' : 'http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu' ,
532 'id' : '150615_telabyad_kentin_cogu' ,
534 'title' : "YPG: Tel Abyad'ın tamamı kontrolümüzde" ,
536 'timestamp' : 1434397334 ,
537 'upload_date' : '20150615' ,
540 'skip_download' : True ,
543 # single video embedded with mediaAssetPage.init() (regional section)
544 'url' : 'http://www.bbc.com/mundo/video_fotos/2015/06/150619_video_honduras_militares_hospitales_corrupcion_aw' ,
546 'id' : '150619_video_honduras_militares_hospitales_corrupcion_aw' ,
548 'title' : 'Honduras militariza sus hospitales por nuevo escándalo de corrupción' ,
550 'timestamp' : 1434713142 ,
551 'upload_date' : '20150619' ,
554 'skip_download' : True ,
557 # single video from video playlist embedded with vxp-playlist-data JSON
558 'url' : 'http://www.bbc.com/news/video_and_audio/must_see/33376376' ,
562 'title' : '''Judge Mindy Glazer: "I'm sorry to see you here... I always wondered what happened to you"''' ,
566 'skip_download' : True ,
569 # single video story with digitalData
570 'url' : 'http://www.bbc.com/travel/story/20150625-sri-lankas-spicy-secret' ,
574 'title' : 'Sri Lanka’s spicy secret' ,
575 'description' : 'As a new train line to Jaffna opens up the country’s north, travellers can experience a truly distinct slice of Tamil culture.' ,
576 'timestamp' : 1437674293 ,
577 'upload_date' : '20150723' ,
581 'skip_download' : True ,
584 # single video story without digitalData
585 'url' : 'http://www.bbc.com/autos/story/20130513-hyundais-rock-star' ,
589 'title' : 'Hyundai Santa Fe Sport: Rock star' ,
590 'description' : 'md5:b042a26142c4154a6e472933cf20793d' ,
591 'timestamp' : 1368473503 ,
592 'upload_date' : '20130513' ,
596 'skip_download' : True ,
599 # single video with playlist.sxml URL
600 'url' : 'http://www.bbc.com/sport/0/football/33653409' ,
604 'title' : 'Transfers: Cristiano Ronaldo to Man Utd, Arsenal to spend?' ,
605 'description' : 'md5:398fca0e2e701c609d726e034fa1fc89' ,
610 'skip_download' : True ,
613 # article with multiple videos embedded with playlist.sxml
614 'url' : 'http://www.bbc.com/sport/0/football/34475836' ,
617 'title' : 'What Liverpool can expect from Klopp' ,
621 # single video with playlist URL from weather section
622 'url' : 'http://www.bbc.com/weather/features/33601775' ,
623 'only_matching' : True ,
625 # custom redirection to www.bbc.com
626 'url' : 'http://www.bbc.co.uk/news/science-environment-33661876' ,
627 'only_matching' : True ,
631 def suitable ( cls
, url
):
632 return False if BBCCoUkIE
. suitable ( url
) else super ( BBCIE
, cls
). suitable ( url
)
634 def _extract_from_media_meta ( self
, media_meta
, video_id
):
635 # Direct links to media in media metadata (e.g.
636 # http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu)
637 # TODO: there are also f4m and m3u8 streams incorporated in playlist.sxml
638 source_files
= media_meta
. get ( 'sourceFiles' )
642 'format_id' : format_id
,
643 'ext' : f
. get ( 'encoding' ),
644 'tbr' : float_or_none ( f
. get ( 'bitrate' ), 1000 ),
645 'filesize' : int_or_none ( f
. get ( 'filesize' )),
646 } for format_id
, f
in source_files
. items () if f
. get ( 'url' )], []
648 programme_id
= media_meta
. get ( 'externalId' )
650 return self
._ download
_ media
_ selector
( programme_id
)
652 # Process playlist.sxml as legacy playlist
653 href
= media_meta
. get ( 'href' )
655 playlist
= self
._ download
_l egacy
_ playlist
_u rl
( href
)
656 _
, _
, _
, _
, formats
, subtitles
= self
._ extract
_ from
_l egacy
_ playlist
( playlist
, video_id
)
657 return formats
, subtitles
661 def _extract_from_playlist_sxml ( self
, url
, playlist_id
, timestamp
):
662 programme_id
, title
, description
, duration
, formats
, subtitles
= \
663 self
._ process
_l egacy
_ playlist
_u rl
( url
, playlist_id
)
664 self
._ sort
_ formats
( formats
)
668 'description' : description
,
669 'duration' : duration
,
670 'timestamp' : timestamp
,
672 'subtitles' : subtitles
,
675 def _real_extract ( self
, url
):
676 playlist_id
= self
._ match
_ id
( url
)
678 webpage
= self
._ download
_ webpage
( url
, playlist_id
)
680 timestamp
= parse_iso8601 ( self
._ search
_ regex
(
681 [ r
'"datePublished":\s*"([^"]+)' ,
682 r
'<meta[^>]+property="article:published_time"[^>]+content="([^"]+)"' ,
683 r
'itemprop="datePublished"[^>]+datetime="([^"]+)"' ],
684 webpage
, 'date' , default
= None ))
686 # article with multiple videos embedded with playlist.sxml (e.g.
687 # http://www.bbc.com/sport/0/football/34475836)
688 playlists
= re
. findall ( r
'<param[^>]+name="playlist"[^>]+value="([^"]+)"' , webpage
)
691 self
._ extract
_ from
_ playlist
_ sxml
( playlist_url
, playlist_id
, timestamp
)
692 for playlist_url
in playlists
]
693 playlist_title
= self
._ og
_ search
_ title
( webpage
)
694 playlist_description
= self
._ og
_ search
_ description
( webpage
, default
= None )
695 return self
. playlist_result ( entries
, playlist_id
, playlist_title
, playlist_description
)
697 # single video story (e.g. http://www.bbc.com/travel/story/20150625-sri-lankas-spicy-secret)
698 programme_id
= self
._ search
_ regex
(
699 [ r
'data-video-player-vpid="([\da-z] {8} )"' ,
700 r
'<param[^>]+name="externalIdentifier"[^>]+value="([\da-z] {8} )"' ],
701 webpage
, 'vpid' , default
= None )
705 # single video in news article embedded with data-playable (e.g.
706 # http://www.bbc.com/news/world-us-canada-34473351)
707 data_playable
= self
._ parse
_ json
(
708 unescapeHTML ( self
._ search
_ regex
(
709 r
'data-playable="( {.+?} )"' , webpage
, 'data playable' , default
= '{}' )),
710 programme_id
, fatal
= False )
712 # data-playable has video vpid in settings.playlistObject.items (e.g.
713 # http://www.bbc.com/news/world-us-canada-34473351)
714 settings
= data_playable
. get ( 'settings' , {})
716 playlist_object
= settings
. get ( 'playlistObject' , {})
718 items
= playlist_object
. get ( 'items' )
719 if items
and isinstance ( items
, list ):
720 duration
= int_or_none ( items
[ 0 ]. get ( 'duration' ))
721 programme_id
= items
[ 0 ]. get ( 'vpid' )
723 # data-playable has no vpid but has a playlist.sxml URLs
724 # in otherSettings.playlist (e.g.
725 # http://www.bbc.com/turkce/multimedya/2015/10/151010_vid_ankara_patlama_ani)
726 playlist
= data_playable
. get ( 'otherSettings' , {}).get('playlist', {}
)
728 return self
._ extract
_ from
_ playlist
_ sxml
(
729 playlist
. get ( 'progressiveDownloadUrl' ), playlist_id
, timestamp
)
732 formats
, subtitles
= self
._ download
_ media
_ selector
( programme_id
)
733 self
._ sort
_ formats
( formats
)
734 # digitalData may be missing (e.g. http://www.bbc.com/autos/story/20130513-hyundais-rock-star)
735 digital_data
= self
._ parse
_ json
(
737 r
'var\s+digitalData\s*=\s*( {.+?} );?\n' , webpage
, 'digital data' , default
= '{}' ),
738 programme_id
, fatal
= False )
739 page_info
= digital_data
. get ( 'page' , {}).get('pageInfo', {}
)
740 title
= page_info
. get ( 'pageName' ) or self
._ og
_ search
_ title
( webpage
)
741 description
= page_info
. get ( 'description' ) or self
._ og
_ search
_ description
( webpage
)
742 timestamp
= parse_iso8601 ( page_info
. get ( 'publicationDate' )) or timestamp
746 'description' : description
,
747 'timestamp' : timestamp
,
748 'duration' : duration
,
750 'subtitles' : subtitles
,
753 playlist_title
= self
._ html
_ search
_ regex
(
754 r
'<title>(.*?)(?:\s*-\s*BBC [^ ]+)?</title>' , webpage
, 'playlist title' )
755 playlist_description
= self
._ og
_ search
_ description
( webpage
, default
= None )
757 def extract_all ( pattern
):
758 return list ( filter ( None , map (
759 lambda s
: self
._ parse
_ json
( s
, playlist_id
, fatal
= False ),
760 re
. findall ( pattern
, webpage
))))
762 # Multiple video article (e.g.
763 # http://www.bbc.co.uk/blogs/adamcurtis/entries/3662a707-0af9-3149-963f-47bea720b460)
764 EMBED_URL
= r
'https?://(?:www\.)?bbc\.co\.uk/(?:[^/]+/)+[\da-z] {8} (?:\b[^"]+)?'
766 for match
in extract_all ( r
'new\s+SMP\(( {.+?} )\)' ):
767 embed_url
= match
. get ( 'playerSettings' , {}). get ( 'externalEmbedUrl' )
768 if embed_url
and re
. match ( EMBED_URL
, embed_url
):
769 entries
. append ( embed_url
)
770 entries
. extend ( re
. findall (
771 r
'setPlaylist\("( %s )"\)' % EMBED_URL
, webpage
))
773 return self
. playlist_result (
774 [ self
. url_result ( entry
, 'BBCCoUk' ) for entry
in entries
],
775 playlist_id
, playlist_title
, playlist_description
)
777 # Multiple video article (e.g. http://www.bbc.com/news/world-europe-32668511)
778 medias
= extract_all ( r
"data-media-meta='( {[^']+} )'" )
781 # Single video article (e.g. http://www.bbc.com/news/video_and_audio/international)
782 media_asset
= self
._ search
_ regex
(
783 r
'mediaAssetPage\.init\(\s*( {.+?} ), "/' ,
784 webpage
, 'media asset' , default
= None )
786 media_asset_page
= self
._ parse
_ json
( media_asset
, playlist_id
, fatal
= False )
788 for video
in media_asset_page
. get ( 'videos' , {}). values ():
789 medias
. extend ( video
. values ())
792 # Multiple video playlist with single `now playing` entry (e.g.
793 # http://www.bbc.com/news/video_and_audio/must_see/33767813)
794 vxp_playlist
= self
._ parse
_ json
(
796 r
'<script[^>]+class="vxp-playlist-data"[^>]+type="application/json"[^>]*>([^<]+)</script>' ,
797 webpage
, 'playlist data' ),
800 for item
in vxp_playlist
:
801 media
= item
. get ( 'media' )
804 playlist_medias
. append ( media
)
805 # Download single video if found media with asset id matching the video id from URL
806 if item
. get ( 'advert' , {}). get ( 'assetId' ) == playlist_id
:
809 # Fallback to the whole playlist
811 medias
= playlist_medias
814 for num
, media_meta
in enumerate ( medias
, start
= 1 ):
815 formats
, subtitles
= self
._ extract
_ from
_ media
_ meta
( media_meta
, playlist_id
)
818 self
._ sort
_ formats
( formats
)
820 video_id
= media_meta
. get ( 'externalId' )
822 video_id
= playlist_id
if len ( medias
) == 1 else ' %s-%s ' % ( playlist_id
, num
)
824 title
= media_meta
. get ( 'caption' )
826 title
= playlist_title
if len ( medias
) == 1 else ' %s - Video %s ' % ( playlist_title
, num
)
828 duration
= int_or_none ( media_meta
. get ( 'durationInSeconds' )) or parse_duration ( media_meta
. get ( 'duration' ))
831 for image
in media_meta
. get ( 'images' , {}). values ():
832 images
. extend ( image
. values ())
833 if 'image' in media_meta
:
834 images
. append ( media_meta
[ 'image' ])
837 'url' : image
. get ( 'href' ),
838 'width' : int_or_none ( image
. get ( 'width' )),
839 'height' : int_or_none ( image
. get ( 'height' )),
840 } for image
in images
]
845 'thumbnails' : thumbnails
,
846 'duration' : duration
,
847 'timestamp' : timestamp
,
849 'subtitles' : subtitles
,
852 return self
. playlist_result ( entries
, playlist_id
, playlist_title
, playlist_description
)