]>
jfr.im git - yt-dlp.git/blob - youtube_dl/extractor/bbc.py
2 from __future__
import unicode_literals
5 import xml
. etree
. ElementTree
7 from . common
import InfoExtractor
17 from .. compat
import compat_HTTPError
20 class BBCCoUkIE ( InfoExtractor
):
22 IE_DESC
= 'BBC iPlayer'
23 _VALID_URL
= r
'https?://(?:www\.)?bbc\.co\.uk/(?:(?:(?:programmes|iplayer(?:/[^/]+)?/(?:episode|playlist))/)|music/clips[/#])(?P<id>[\da-z] {8} )'
25 _MEDIASELECTOR_URLS
= [
26 # Provides HQ HLS streams with even better quality that pc mediaset but fails
27 # with geolocation in some cases when it's even not geo restricted at all (e.g.
28 # http://www.bbc.co.uk/programmes/b06bp7lf)
29 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/iptv-all/vpid/ %s ' ,
30 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/ %s ' ,
33 _MEDIASELECTION_NS
= 'http://bbc.co.uk/2008/mp/mediaselection'
34 _EMP_PLAYLIST_NS
= 'http://bbc.co.uk/2008/emp/playlist'
43 'url' : 'http://www.bbc.co.uk/programmes/b039g8p7' ,
47 'title' : 'Kaleidoscope, Leonard Cohen' ,
48 'description' : 'The Canadian poet and songwriter reflects on his musical career.' ,
53 'skip_download' : True ,
57 'url' : 'http://www.bbc.co.uk/iplayer/episode/b00yng5w/The_Man_in_Black_Series_3_The_Printed_Name/' ,
61 'title' : 'The Man in Black: Series 3: The Printed Name' ,
62 'description' : "Mark Gatiss introduces Nicholas Pierpan's chilling tale of a writer's devilish pact with a mysterious man. Stars Ewan Bailey." ,
67 'skip_download' : True ,
69 'skip' : 'Episode is no longer available on BBC iPlayer Radio' ,
72 'url' : 'http://www.bbc.co.uk/iplayer/episode/b03vhd1f/The_Voice_UK_Series_3_Blind_Auditions_5/' ,
76 'title' : 'The Voice UK: Series 3: Blind Auditions 5' ,
77 'description' : "Emma Willis and Marvin Humes present the fifth set of blind auditions in the singing competition, as the coaches continue to build their teams based on voice alone." ,
82 'skip_download' : True ,
84 'skip' : 'Currently BBC iPlayer TV programmes are available to play in the UK only' ,
87 'url' : 'http://www.bbc.co.uk/iplayer/episode/p026c7jt/tomorrows-worlds-the-unearthly-history-of-science-fiction-2-invasion' ,
91 'title' : "Tomorrow's Worlds: The Unearthly History of Science Fiction" ,
92 'description' : '2. Invasion' ,
97 'skip_download' : True ,
99 'skip' : 'Currently BBC iPlayer TV programmes are available to play in the UK only' ,
101 'url' : 'http://www.bbc.co.uk/programmes/b04v20dw' ,
105 'title' : 'Pete Tong, The Essential New Tune Special' ,
106 'description' : "Pete has a very special mix - all of 2014's Essential New Tunes!" ,
111 'skip_download' : True ,
114 'url' : 'http://www.bbc.co.uk/music/clips/p02frcc3' ,
119 'title' : 'Pete Tong, Past, Present and Future Special, Madeon - After Hours mix' ,
120 'description' : 'French house superstar Madeon takes us out of the club and onto the after party.' ,
125 'skip_download' : True ,
128 'url' : 'http://www.bbc.co.uk/music/clips/p025c0zz' ,
133 'title' : 'Reading and Leeds Festival, 2014, Rae Morris - Closer (Live on BBC Three)' ,
134 'description' : 'Rae Morris performs Closer for BBC Three at Reading 2014' ,
139 'skip_download' : True ,
142 'url' : 'http://www.bbc.co.uk/iplayer/episode/b054fn09/ad/natural-world-20152016-2-super-powered-owls' ,
146 'title' : 'Natural World, 2015-2016: 2. Super Powered Owls' ,
147 'description' : 'md5:e4db5c937d0e95a7c6b5e654d429183d' ,
152 'skip_download' : True ,
154 'skip' : 'geolocation' ,
156 'url' : 'http://www.bbc.co.uk/iplayer/episode/b05zmgwn/royal-academy-summer-exhibition' ,
160 'description' : 'Kirsty Wark and Morgan Quaintance visit the Royal Academy as it prepares for its annual artistic extravaganza, meeting people who have come together to make the show unique.' ,
161 'title' : 'Royal Academy Summer Exhibition' ,
166 'skip_download' : True ,
168 'skip' : 'geolocation' ,
170 # iptv-all mediaset fails with geolocation however there is no geo restriction
171 # for this programme at all
172 'url' : 'http://www.bbc.co.uk/programmes/b06bp7lf' ,
176 'title' : "Annie Mac's Friday Night, B.Traits sits in for Annie" ,
177 'description' : 'B.Traits sits in for Annie Mac with a Mini-Mix from Disclosure.' ,
182 'skip_download' : True ,
185 'url' : 'http://www.bbc.co.uk/iplayer/playlist/p01dvks4' ,
186 'only_matching' : True ,
188 'url' : 'http://www.bbc.co.uk/music/clips#p02frcc3' ,
189 'only_matching' : True ,
191 'url' : 'http://www.bbc.co.uk/iplayer/cbeebies/episode/b0480276/bing-14-atchoo' ,
192 'only_matching' : True ,
196 class MediaSelectionError ( Exception ):
197 def __init__ ( self
, id ):
200 def _extract_asx_playlist ( self
, connection
, programme_id
):
201 asx
= self
._ download
_ xml
( connection
. get ( 'href' ), programme_id
, 'Downloading ASX playlist' )
202 return [ ref
. get ( 'href' ) for ref
in asx
. findall ( './Entry/ref' )]
204 def _extract_connection ( self
, connection
, programme_id
):
206 kind
= connection
. get ( 'kind' )
207 protocol
= connection
. get ( 'protocol' )
208 supplier
= connection
. get ( 'supplier' )
209 if protocol
== 'http' :
210 href
= connection
. get ( 'href' )
211 transfer_format
= connection
. get ( 'transferFormat' )
213 if supplier
== 'asx' :
214 for i
, ref
in enumerate ( self
._ extract
_ asx
_ playlist
( connection
, programme_id
)):
217 'format_id' : 'ref %s _ %s ' % ( i
, supplier
),
219 # Skip DASH until supported
220 elif transfer_format
== 'dash' :
222 elif transfer_format
== 'hls' :
223 m3u8_formats
= self
._ extract
_ m
3u8_ formats
(
224 href
, programme_id
, ext
= 'mp4' , entry_protocol
= 'm3u8_native' ,
225 m3u8_id
= supplier
, fatal
= False )
227 formats
. extend ( m3u8_formats
)
232 'format_id' : supplier
or kind
or protocol
,
234 elif protocol
== 'rtmp' :
235 application
= connection
. get ( 'application' , 'ondemand' )
236 auth_string
= connection
. get ( 'authString' )
237 identifier
= connection
. get ( 'identifier' )
238 server
= connection
. get ( 'server' )
240 'url' : ' %s :// %s / %s ? %s ' % ( protocol
, server
, application
, auth_string
),
241 'play_path' : identifier
,
242 'app' : ' %s ? %s ' % ( application
, auth_string
),
243 'page_url' : 'http://www.bbc.co.uk' ,
244 'player_url' : 'http://www.bbc.co.uk/emp/releases/iplayer/revisions/617463_618125_4/617463_618125_4_emp.swf' ,
247 'format_id' : supplier
,
251 def _extract_items ( self
, playlist
):
252 return playlist
. findall ( './ {%s} item' % self
._ EMP
_ PLAYLIST
_ NS
)
254 def _findall_ns ( self
, element
, xpath
):
256 for ns
in self
._ NAMESPACES
:
257 elements
. extend ( element
. findall ( xpath
% ns
))
260 def _extract_medias ( self
, media_selection
):
261 error
= media_selection
. find ( './ {%s} error' % self
._ MEDIASELECTION
_ NS
)
263 media_selection
. find ( './ {%s} error' % self
._ EMP
_ PLAYLIST
_ NS
)
264 if error
is not None :
265 raise BBCCoUkIE
. MediaSelectionError ( error
. get ( 'id' ))
266 return self
._ findall
_ ns
( media_selection
, './ {%s} media' )
268 def _extract_connections ( self
, media
):
269 return self
._ findall
_ ns
( media
, './ {%s} connection' )
271 def _extract_video ( self
, media
, programme_id
):
273 vbr
= int_or_none ( media
. get ( 'bitrate' ))
274 vcodec
= media
. get ( 'encoding' )
275 service
= media
. get ( 'service' )
276 width
= int_or_none ( media
. get ( 'width' ))
277 height
= int_or_none ( media
. get ( 'height' ))
278 file_size
= int_or_none ( media
. get ( 'media_file_size' ))
279 for connection
in self
._ extract
_ connections
( media
):
280 conn_formats
= self
._ extract
_ connection
( connection
, programme_id
)
281 for format
in conn_formats
:
287 'filesize' : file_size
,
290 format
[ 'format_id' ] = ' %s _ %s ' % ( service
, format
[ 'format_id' ])
291 formats
. extend ( conn_formats
)
294 def _extract_audio ( self
, media
, programme_id
):
296 abr
= int_or_none ( media
. get ( 'bitrate' ))
297 acodec
= media
. get ( 'encoding' )
298 service
= media
. get ( 'service' )
299 for connection
in self
._ extract
_ connections
( media
):
300 conn_formats
= self
._ extract
_ connection
( connection
, programme_id
)
301 for format
in conn_formats
:
303 'format_id' : ' %s _ %s ' % ( service
, format
[ 'format_id' ]),
307 formats
. extend ( conn_formats
)
310 def _get_subtitles ( self
, media
, programme_id
):
312 for connection
in self
._ extract
_ connections
( media
):
313 captions
= self
._ download
_ xml
( connection
. get ( 'href' ), programme_id
, 'Downloading captions' )
314 lang
= captions
. get ( ' {http://www.w3.org/XML/1998/namespace} lang' , 'en' )
317 'url' : connection
. get ( 'href' ),
323 def _raise_extractor_error ( self
, media_selection_error
):
324 raise ExtractorError (
325 ' %s returned error: %s ' % ( self
. IE_NAME
, media_selection_error
. id ),
328 def _download_media_selector ( self
, programme_id
):
329 last_exception
= None
330 for mediaselector_url
in self
._ MEDIASELECTOR
_U RLS
:
332 return self
._ download
_ media
_ selector
_u rl
(
333 mediaselector_url
% programme_id
, programme_id
)
334 except BBCCoUkIE
. MediaSelectionError
as e
:
335 if e
. id in ( 'notukerror' , 'geolocation' ):
338 self
._ raise
_ extractor
_ error
( e
)
339 self
._ raise
_ extractor
_ error
( last_exception
)
341 def _download_media_selector_url ( self
, url
, programme_id
= None ):
343 media_selection
= self
._ download
_ xml
(
344 url
, programme_id
, 'Downloading media selection XML' )
345 except ExtractorError
as ee
:
346 if isinstance ( ee
. cause
, compat_HTTPError
) and ee
. cause
. code
== 403 :
347 media_selection
= xml
. etree
. ElementTree
. fromstring ( ee
. cause
. read (). decode ( 'utf-8' ))
350 return self
._ process
_ media
_ selector
( media_selection
, programme_id
)
352 def _process_media_selector ( self
, media_selection
, programme_id
):
356 for media
in self
._ extract
_ medias
( media_selection
):
357 kind
= media
. get ( 'kind' )
359 formats
. extend ( self
._ extract
_ audio
( media
, programme_id
))
360 elif kind
== 'video' :
361 formats
. extend ( self
._ extract
_ video
( media
, programme_id
))
362 elif kind
== 'captions' :
363 subtitles
= self
. extract_subtitles ( media
, programme_id
)
364 return formats
, subtitles
366 def _download_playlist ( self
, playlist_id
):
368 playlist
= self
._ download
_ json
(
369 'http://www.bbc.co.uk/programmes/ %s /playlist.json' % playlist_id
,
370 playlist_id
, 'Downloading playlist JSON' )
372 version
= playlist
. get ( 'defaultAvailableVersion' )
374 smp_config
= version
[ 'smpConfig' ]
375 title
= smp_config
[ 'title' ]
376 description
= smp_config
[ 'summary' ]
377 for item
in smp_config
[ 'items' ]:
379 if kind
!= 'programme' and kind
!= 'radioProgramme' :
381 programme_id
= item
. get ( 'vpid' )
382 duration
= int_or_none ( item
. get ( 'duration' ))
383 formats
, subtitles
= self
._ download
_ media
_ selector
( programme_id
)
384 return programme_id
, title
, description
, duration
, formats
, subtitles
385 except ExtractorError
as ee
:
386 if not ( isinstance ( ee
. cause
, compat_HTTPError
) and ee
. cause
. code
== 404 ):
389 # fallback to legacy playlist
390 return self
._ process
_l egacy
_ playlist
( playlist_id
)
392 def _process_legacy_playlist_url ( self
, url
, display_id
):
393 playlist
= self
._ download
_l egacy
_ playlist
_u rl
( url
, display_id
)
394 return self
._ extract
_ from
_l egacy
_ playlist
( playlist
, display_id
)
396 def _process_legacy_playlist ( self
, playlist_id
):
397 return self
._ process
_l egacy
_ playlist
_u rl
(
398 'http://www.bbc.co.uk/iplayer/playlist/ %s ' % playlist_id
, playlist_id
)
400 def _download_legacy_playlist_url ( self
, url
, playlist_id
= None ):
401 return self
._ download
_ xml
(
402 url
, playlist_id
, 'Downloading legacy playlist XML' )
404 def _extract_from_legacy_playlist ( self
, playlist
, playlist_id
):
405 no_items
= playlist
. find ( './ {%s} noItems' % self
._ EMP
_ PLAYLIST
_ NS
)
406 if no_items
is not None :
407 reason
= no_items
. get ( 'reason' )
408 if reason
== 'preAvailability' :
409 msg
= 'Episode %s is not yet available' % playlist_id
410 elif reason
== 'postAvailability' :
411 msg
= 'Episode %s is no longer available' % playlist_id
412 elif reason
== 'noMedia' :
413 msg
= 'Episode %s is not currently available' % playlist_id
415 msg
= 'Episode %s is not available: %s ' % ( playlist_id
, reason
)
416 raise ExtractorError ( msg
, expected
= True )
418 for item
in self
._ extract
_ items
( playlist
):
419 kind
= item
. get ( 'kind' )
420 if kind
!= 'programme' and kind
!= 'radioProgramme' :
422 title
= playlist
. find ( './ {%s} title' % self
._ EMP
_ PLAYLIST
_ NS
). text
423 description_el
= playlist
. find ( './ {%s} summary' % self
._ EMP
_ PLAYLIST
_ NS
)
424 description
= description_el
. text
if description_el
is not None else None
426 def get_programme_id ( item
):
427 def get_from_attributes ( item
):
428 for p
in ( 'identifier' , 'group' ):
430 if value
and re
. match ( r
'^[pb][\da-z] {7} $' , value
):
432 get_from_attributes ( item
)
433 mediator
= item
. find ( './ {%s} mediator' % self
._ EMP
_ PLAYLIST
_ NS
)
434 if mediator
is not None :
435 return get_from_attributes ( mediator
)
437 programme_id
= get_programme_id ( item
)
438 duration
= int_or_none ( item
. get ( 'duration' ))
441 formats
, subtitles
= self
._ download
_ media
_ selector
( programme_id
)
443 formats
, subtitles
= self
._ process
_ media
_ selector
( item
, playlist_id
)
444 programme_id
= playlist_id
446 return programme_id
, title
, description
, duration
, formats
, subtitles
448 def _real_extract ( self
, url
):
449 group_id
= self
._ match
_ id
( url
)
451 webpage
= self
._ download
_ webpage
( url
, group_id
, 'Downloading video page' )
455 tviplayer
= self
._ search
_ regex
(
456 r
'mediator\.bind\(( {.+?} )\s*,\s*document\.getElementById' ,
457 webpage
, 'player' , default
= None )
460 player
= self
._ parse
_ json
( tviplayer
, group_id
). get ( 'player' , {})
461 duration
= int_or_none ( player
. get ( 'duration' ))
462 programme_id
= player
. get ( 'vpid' )
465 programme_id
= self
._ search
_ regex
(
466 r
'"vpid"\s*:\s*"([\da-z] {8} )"' , webpage
, 'vpid' , fatal
= False , default
= None )
469 formats
, subtitles
= self
._ download
_ media
_ selector
( programme_id
)
470 title
= self
._ og
_ search
_ title
( webpage
)
471 description
= self
._ search
_ regex
(
472 r
'<p class="[^"]*medium-description[^"]*">([^<]+)</p>' ,
473 webpage
, 'description' , fatal
= False )
475 programme_id
, title
, description
, duration
, formats
, subtitles
= self
._ download
_ playlist
( group_id
)
477 self
._ sort
_ formats
( formats
)
482 'description' : description
,
483 'thumbnail' : self
._ og
_ search
_ thumbnail
( webpage
, default
= None ),
484 'duration' : duration
,
486 'subtitles' : subtitles
,
490 class BBCIE ( BBCCoUkIE
):
493 _VALID_URL
= r
'https?://(?:www\.)?bbc\.(?:com|co\.uk)/(?:[^/]+/)+(?P<id>[^/#?]+)'
495 _MEDIASELECTOR_URLS
= [
496 # Provides HQ HLS streams but fails with geolocation in some cases when it's
497 # even not geo restricted at all
498 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/iptv-all/vpid/ %s ' ,
499 # Provides more formats, namely direct mp4 links, but fails on some videos with
500 # notukerror for non UK (?) users (e.g.
501 # http://www.bbc.com/travel/story/20150625-sri-lankas-spicy-secret)
502 'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/ %s ' ,
503 # Provides fewer formats, but works everywhere for everybody (hopefully)
504 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/journalism-pc/vpid/ %s ' ,
508 # article with multiple videos embedded with data-playable containing vpids
509 'url' : 'http://www.bbc.com/news/world-europe-32668511' ,
511 'id' : 'world-europe-32668511' ,
512 'title' : 'Russia stages massive WW2 parade despite Western boycott' ,
513 'description' : 'md5:00ff61976f6081841f759a08bf78cc9c' ,
517 # article with multiple videos embedded with data-playable (more videos)
518 'url' : 'http://www.bbc.com/news/business-28299555' ,
520 'id' : 'business-28299555' ,
521 'title' : 'Farnborough Airshow: Video highlights' ,
522 'description' : 'BBC reports and video highlights at the Farnborough Airshow.' ,
527 # article with multiple videos embedded with `new SMP()`
529 'url' : 'http://www.bbc.co.uk/blogs/adamcurtis/entries/3662a707-0af9-3149-963f-47bea720b460' ,
531 'id' : '3662a707-0af9-3149-963f-47bea720b460' ,
532 'title' : 'BBC Blogs - Adam Curtis - BUGGER' ,
534 'playlist_count' : 18 ,
536 # single video embedded with data-playable containing vpid
537 'url' : 'http://www.bbc.com/news/world-europe-32041533' ,
541 'title' : 'Aerial footage showed the site of the crash in the Alps - courtesy BFM TV' ,
542 'description' : 'md5:2868290467291b37feda7863f7a83f54' ,
544 'timestamp' : 1427219242 ,
545 'upload_date' : '20150324' ,
549 'skip_download' : True ,
552 # article with single video embedded with data-playable containing XML playlist
553 # with direct video links as progressiveDownloadUrl (for now these are extracted)
554 # and playlist with f4m and m3u8 as streamingUrl
555 'url' : 'http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu' ,
557 'id' : '150615_telabyad_kentin_cogu' ,
559 'title' : "YPG: Tel Abyad'ın tamamı kontrolümüzde" ,
560 'timestamp' : 1434397334 ,
561 'upload_date' : '20150615' ,
564 'skip_download' : True ,
567 # single video embedded with data-playable containing XML playlists (regional section)
568 'url' : 'http://www.bbc.com/mundo/video_fotos/2015/06/150619_video_honduras_militares_hospitales_corrupcion_aw' ,
570 'id' : '150619_video_honduras_militares_hospitales_corrupcion_aw' ,
572 'title' : 'Honduras militariza sus hospitales por nuevo escándalo de corrupción' ,
573 'timestamp' : 1434713142 ,
574 'upload_date' : '20150619' ,
577 'skip_download' : True ,
580 # single video from video playlist embedded with vxp-playlist-data JSON
581 'url' : 'http://www.bbc.com/news/video_and_audio/must_see/33376376' ,
585 'title' : '''Judge Mindy Glazer: "I'm sorry to see you here... I always wondered what happened to you"''' ,
589 'skip_download' : True ,
592 # single video story with digitalData
593 'url' : 'http://www.bbc.com/travel/story/20150625-sri-lankas-spicy-secret' ,
597 'title' : 'Sri Lanka’s spicy secret' ,
598 'description' : 'As a new train line to Jaffna opens up the country’s north, travellers can experience a truly distinct slice of Tamil culture.' ,
599 'timestamp' : 1437674293 ,
600 'upload_date' : '20150723' ,
604 'skip_download' : True ,
607 # single video story without digitalData
608 'url' : 'http://www.bbc.com/autos/story/20130513-hyundais-rock-star' ,
612 'title' : 'Hyundai Santa Fe Sport: Rock star' ,
613 'description' : 'md5:b042a26142c4154a6e472933cf20793d' ,
614 'timestamp' : 1415867444 ,
615 'upload_date' : '20141113' ,
619 'skip_download' : True ,
622 # single video with playlist.sxml URL in playlist param
623 'url' : 'http://www.bbc.com/sport/0/football/33653409' ,
627 'title' : 'Transfers: Cristiano Ronaldo to Man Utd, Arsenal to spend?' ,
628 'description' : 'BBC Sport \' s David Ornstein has the latest transfer gossip, including rumours of a Manchester United return for Cristiano Ronaldo.' ,
633 'skip_download' : True ,
636 # article with multiple videos embedded with playlist.sxml in playlist param
637 'url' : 'http://www.bbc.com/sport/0/football/34475836' ,
640 'title' : 'What Liverpool can expect from Klopp' ,
644 # single video with playlist URL from weather section
645 'url' : 'http://www.bbc.com/weather/features/33601775' ,
646 'only_matching' : True ,
648 # custom redirection to www.bbc.com
649 'url' : 'http://www.bbc.co.uk/news/science-environment-33661876' ,
650 'only_matching' : True ,
654 def suitable ( cls
, url
):
655 return False if BBCCoUkIE
. suitable ( url
) else super ( BBCIE
, cls
). suitable ( url
)
657 def _extract_from_media_meta ( self
, media_meta
, video_id
):
658 # Direct links to media in media metadata (e.g.
659 # http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu)
660 # TODO: there are also f4m and m3u8 streams incorporated in playlist.sxml
661 source_files
= media_meta
. get ( 'sourceFiles' )
665 'format_id' : format_id
,
666 'ext' : f
. get ( 'encoding' ),
667 'tbr' : float_or_none ( f
. get ( 'bitrate' ), 1000 ),
668 'filesize' : int_or_none ( f
. get ( 'filesize' )),
669 } for format_id
, f
in source_files
. items () if f
. get ( 'url' )], []
671 programme_id
= media_meta
. get ( 'externalId' )
673 return self
._ download
_ media
_ selector
( programme_id
)
675 # Process playlist.sxml as legacy playlist
676 href
= media_meta
. get ( 'href' )
678 playlist
= self
._ download
_l egacy
_ playlist
_u rl
( href
)
679 _
, _
, _
, _
, formats
, subtitles
= self
._ extract
_ from
_l egacy
_ playlist
( playlist
, video_id
)
680 return formats
, subtitles
684 def _extract_from_playlist_sxml ( self
, url
, playlist_id
, timestamp
):
685 programme_id
, title
, description
, duration
, formats
, subtitles
= \
686 self
._ process
_l egacy
_ playlist
_u rl
( url
, playlist_id
)
687 self
._ sort
_ formats
( formats
)
691 'description' : description
,
692 'duration' : duration
,
693 'timestamp' : timestamp
,
695 'subtitles' : subtitles
,
698 def _real_extract ( self
, url
):
699 playlist_id
= self
._ match
_ id
( url
)
701 webpage
= self
._ download
_ webpage
( url
, playlist_id
)
704 playlist_title
= None
705 playlist_description
= None
707 ld
= self
._ parse
_ json
(
709 r
'(?s)<script type="application/ld\+json">(.+?)</script>' ,
710 webpage
, 'ld json' , default
= '{}' ),
711 playlist_id
, fatal
= False )
713 timestamp
= parse_iso8601 ( ld
. get ( 'datePublished' ))
714 playlist_title
= ld
. get ( 'headline' )
715 playlist_description
= ld
. get ( 'articleBody' )
718 timestamp
= parse_iso8601 ( self
._ search
_ regex
(
719 [ r
'<meta[^>]+property="article:published_time"[^>]+content="([^"]+)"' ,
720 r
'itemprop="datePublished"[^>]+datetime="([^"]+)"' ,
721 r
'"datePublished":\s*"([^"]+)' ],
722 webpage
, 'date' , default
= None ))
726 # article with multiple videos embedded with playlist.sxml (e.g.
727 # http://www.bbc.com/sport/0/football/34475836)
728 playlists
= re
. findall ( r
'<param[^>]+name="playlist"[^>]+value="([^"]+)"' , webpage
)
731 self
._ extract
_ from
_ playlist
_ sxml
( playlist_url
, playlist_id
, timestamp
)
732 for playlist_url
in playlists
]
734 # news article with multiple videos embedded with data-playable
735 data_playables
= re
. findall ( r
'data-playable=(["\' ])( {.+?}
) \
1 ', webpage)
737 for _, data_playable_json in data_playables:
738 data_playable = self._parse_json(
739 unescapeHTML(data_playable_json), playlist_id, fatal=False)
740 if not data_playable:
742 settings = data_playable.get(' settings
', {})
744 # data-playable with video vpid in settings.playlistObject.items (e.g.
745 # http://www.bbc.com/news/world-us-canada-34473351)
746 playlist_object = settings.get(' playlistObject
', {})
748 items = playlist_object.get(' items
')
749 if items and isinstance(items, list):
750 title = playlist_object[' title
']
751 description = playlist_object.get(' summary
')
752 duration = int_or_none(items[0].get(' duration
'))
753 programme_id = items[0].get(' vpid
')
754 formats, subtitles = self._download_media_selector(programme_id)
755 self._sort_formats(formats)
759 ' description
': description,
760 ' timestamp
': timestamp,
761 ' duration
': duration,
763 ' subtitles
': subtitles,
766 # data-playable without vpid but with a playlist.sxml URLs
767 # in otherSettings.playlist (e.g.
768 # http://www.bbc.com/turkce/multimedya/2015/10/151010_vid_ankara_patlama_ani)
769 playlist = data_playable.get(' otherSettings
', {}).get('playlist', {} )
771 entries.append(self._extract_from_playlist_sxml(
772 playlist.get(' progressiveDownloadUrl
'), playlist_id, timestamp))
775 playlist_title = playlist_title or remove_end(self._og_search_title(webpage), ' - BBC News
')
776 playlist_description = playlist_description or self._og_search_description(webpage, default=None)
777 return self.playlist_result(entries, playlist_id, playlist_title, playlist_description)
779 # single video story (e.g. http://www.bbc.com/travel/story/20150625-sri-lankas-spicy-secret)
780 programme_id = self._search_regex(
781 [r' data
- video
- player
- vpid
= "([\da-z] {8} )" ',
782 r' < param
[ ^
>]+ name
= "externalIdentifier" [ ^
>]+ value
= "([\da-z] {8} )" '],
783 webpage, ' vpid
', default=None)
786 formats, subtitles = self._download_media_selector(programme_id)
787 self._sort_formats(formats)
788 # digitalData may be missing (e.g. http://www.bbc.com/autos/story/20130513-hyundais-rock-star)
789 digital_data = self._parse_json(
791 r' var\s
+ digitalData\s
*= \s
*( {.+?}
); ?
\n ', webpage, ' digital data
', default=' {} '),
792 programme_id, fatal=False)
793 page_info = digital_data.get(' page
', {}).get('pageInfo', {} )
794 title = page_info.get(' pageName
') or self._og_search_title(webpage)
795 description = page_info.get(' description
') or self._og_search_description(webpage)
796 timestamp = parse_iso8601(page_info.get(' publicationDate
')) or timestamp
800 ' description
': description,
801 ' timestamp
': timestamp,
803 ' subtitles
': subtitles,
806 playlist_title = self._html_search_regex(
807 r' < title
>(.* ?
)( ?
: \s
*- \s
* BBC
[ ^
]+) ?
</ title
> ', webpage, ' playlist title
')
808 playlist_description = self._og_search_description(webpage, default=None)
810 def extract_all(pattern):
811 return list(filter(None, map(
812 lambda s: self._parse_json(s, playlist_id, fatal=False),
813 re.findall(pattern, webpage))))
815 # Multiple video article (e.g.
816 # http://www.bbc.co.uk/blogs/adamcurtis/entries/3662a707-0af9-3149-963f-47bea720b460)
817 EMBED_URL = r' https?
://( ?
: www\
.) ?bbc\
. co\
. uk
/( ?
:[ ^
/]+/)+[ \da
- z
] {8}
( ?
: \b [ ^
"]+)?'
819 for match in extract_all(r'new\s+SMP\(( {.+?} )\)'):
820 embed_url = match.get('playerSettings', {}).get('externalEmbedUrl')
821 if embed_url and re.match(EMBED_URL, embed_url):
822 entries.append(embed_url)
823 entries.extend(re.findall(
824 r'setPlaylist\(" ( %s) "\)' % EMBED_URL, webpage))
826 return self.playlist_result(
827 [self.url_result(entry, 'BBCCoUk') for entry in entries],
828 playlist_id, playlist_title, playlist_description)
830 # Multiple video article (e.g. http://www.bbc.com/news/world-europe-32668511)
831 medias = extract_all(r" data
- media
- meta
= '( {[^']+} )' ")
834 # Single video article (e.g. http://www.bbc.com/news/video_and_audio/international)
835 media_asset = self._search_regex(
836 r'mediaAssetPage\.init\(\s*( {.+?} ), " / ',
837 webpage, ' media asset
', default=None)
839 media_asset_page = self._parse_json(media_asset, playlist_id, fatal=False)
841 for video in media_asset_page.get(' videos
', {}).values():
842 medias.extend(video.values())
845 # Multiple video playlist with single `now playing` entry (e.g.
846 # http://www.bbc.com/news/video_and_audio/must_see/33767813)
847 vxp_playlist = self._parse_json(
849 r' < script
[ ^
>]+ class = "vxp-playlist-data" [ ^
>]+ type = "application/json" [ ^
>]*>([ ^
<]+)</ script
> ',
850 webpage, ' playlist data
'),
853 for item in vxp_playlist:
854 media = item.get(' media
')
857 playlist_medias.append(media)
858 # Download single video if found media with asset id matching the video id from URL
859 if item.get(' advert
', {}).get(' assetId
') == playlist_id:
862 # Fallback to the whole playlist
864 medias = playlist_medias
867 for num, media_meta in enumerate(medias, start=1):
868 formats, subtitles = self._extract_from_media_meta(media_meta, playlist_id)
871 self._sort_formats(formats)
873 video_id = media_meta.get(' externalId
')
875 video_id = playlist_id if len(medias) == 1 else ' %s-%s ' % (playlist_id, num)
877 title = media_meta.get(' caption
')
879 title = playlist_title if len(medias) == 1 else ' %s - Video
%s ' % (playlist_title, num)
881 duration = int_or_none(media_meta.get(' durationInSeconds
')) or parse_duration(media_meta.get(' duration
'))
884 for image in media_meta.get(' images
', {}).values():
885 images.extend(image.values())
886 if ' image
' in media_meta:
887 images.append(media_meta[' image
'])
890 ' url
': image.get(' href
'),
891 ' width
': int_or_none(image.get(' width
')),
892 ' height
': int_or_none(image.get(' height
')),
893 } for image in images]
898 ' thumbnails
': thumbnails,
899 ' duration
': duration,
900 ' timestamp
': timestamp,
902 ' subtitles
': subtitles,
905 return self.playlist_result(entries, playlist_id, playlist_title, playlist_description)