[yt-dlp.git] / youtube_dl / extractor / bbccouk.py

from __future__ import unicode_literals

import re

from .subtitles import SubtitlesInfoExtractor
from ..utils import ExtractorError


class BBCCoUkIE(SubtitlesInfoExtractor):
    IE_NAME = 'bbc.co.uk'
    IE_DESC = 'BBC iPlayer'
    _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:programmes|iplayer/episode)/(?P<id>[\da-z]{8})'

    _TESTS = [
        {
            'url': 'http://www.bbc.co.uk/programmes/p01q7wz1',
            'info_dict': {
                'id': 'p01q7wz4',
                'ext': 'flv',
                'title': 'Friction: Blu Mar Ten guest mix: Blu Mar Ten - Guest Mix',
                'description': 'Blu Mar Ten deliver a Guest Mix for Friction.',
                'duration': 1936,
            },
            'params': {
                # rtmp download
                'skip_download': True,
            }
        },
        {
            'url': 'http://www.bbc.co.uk/iplayer/episode/b00yng5w/The_Man_in_Black_Series_3_The_Printed_Name/',
            'info_dict': {
                'id': 'b00yng1d',
                'ext': 'flv',
                'title': 'The Man in Black: Series 3: The Printed Name',
                'description': "Mark Gatiss introduces Nicholas Pierpan's chilling tale of a writer's devilish pact with a mysterious man. Stars Ewan Bailey.",
                'duration': 1800,
            },
            'params': {
                # rtmp download
                'skip_download': True,
            },
            'skip': 'Episode is no longer available on BBC iPlayer Radio',
        },
        {
            'url': 'http://www.bbc.co.uk/iplayer/episode/b03vhd1f/The_Voice_UK_Series_3_Blind_Auditions_5/',
            'info_dict': {
                'id': 'b00yng1d',
                'ext': 'flv',
                'title': 'The Voice UK: Series 3: Blind Auditions 5',
                'description': "Emma Willis and Marvin Humes present the fifth set of blind auditions in the singing competition, as the coaches continue to build their teams based on voice alone.",
                'duration': 5100,
            },
            'params': {
                # rtmp download
                'skip_download': True,
            },
            'skip': 'Currently BBC iPlayer TV programmes are available to play in the UK only',
        }
    ]

    def _extract_asx_playlist(self, connection, programme_id):
        asx = self._download_xml(connection.get('href'), programme_id, 'Downloading ASX playlist')
        return [ref.get('href') for ref in asx.findall('./Entry/ref')]

    def _extract_connection(self, connection, programme_id):
        formats = []
        protocol = connection.get('protocol')
        supplier = connection.get('supplier')
        if protocol == 'http':
            href = connection.get('href')
            # ASX playlist
            if supplier == 'asx':
                for i, ref in enumerate(self._extract_asx_playlist(connection, programme_id)):
                    formats.append({
                        'url': ref,
                        'format_id': 'ref%s_%s' % (i, supplier),
                    })
            # Direct link
            else:
                formats.append({
                    'url': href,
                    'format_id': supplier,
                })
        elif protocol == 'rtmp':
            application = connection.get('application', 'ondemand')
            auth_string = connection.get('authString')
            identifier = connection.get('identifier')
            server = connection.get('server')
            formats.append({
                'url': '%s://%s/%s?%s' % (protocol, server, application, auth_string),
                'play_path': identifier,
                'app': '%s?%s' % (application, auth_string),
                'page_url': 'http://www.bbc.co.uk',
                'player_url': 'http://www.bbc.co.uk/emp/releases/iplayer/revisions/617463_618125_4/617463_618125_4_emp.swf',
                'rtmp_live': False,
                'ext': 'flv',
                'format_id': supplier,
            })
        return formats

    def _extract_items(self, playlist):
        return playlist.findall('./{http://bbc.co.uk/2008/emp/playlist}item')

    def _extract_medias(self, media_selection):
        return media_selection.findall('./{http://bbc.co.uk/2008/mp/mediaselection}media')

    def _extract_connections(self, media):
        return media.findall('./{http://bbc.co.uk/2008/mp/mediaselection}connection')

    def _extract_video(self, media, programme_id):
        formats = []
        vbr = int(media.get('bitrate'))
        vcodec = media.get('encoding')
        service = media.get('service')
        width = int(media.get('width'))
        height = int(media.get('height'))
        file_size = int(media.get('media_file_size'))
        for connection in self._extract_connections(media):
            conn_formats = self._extract_connection(connection, programme_id)
            for format in conn_formats:
                format.update({
                    'format_id': '%s_%s' % (service, format['format_id']),
                    'width': width,
                    'height': height,
                    'vbr': vbr,
                    'vcodec': vcodec,
                    'filesize': file_size,
                })
            formats.extend(conn_formats)
        return formats

    def _extract_audio(self, media, programme_id):
        formats = []
        abr = int(media.get('bitrate'))
        acodec = media.get('encoding')
        service = media.get('service')
        for connection in self._extract_connections(media):
            conn_formats = self._extract_connection(connection, programme_id)
            for format in conn_formats:
                format.update({
                    'format_id': '%s_%s' % (service, format['format_id']),
                    'abr': abr,
                    'acodec': acodec,
                })
            formats.extend(conn_formats)
        return formats

    def _extract_captions(self, media, programme_id):
        subtitles = {}
        for connection in self._extract_connections(media):
            captions = self._download_xml(connection.get('href'), programme_id, 'Downloading captions')
            lang = captions.get('{http://www.w3.org/XML/1998/namespace}lang', 'en')
            ps = captions.findall('./{0}body/{0}div/{0}p'.format('{http://www.w3.org/2006/10/ttaf1}'))
            srt = ''
            for pos, p in enumerate(ps):
                srt += '%s\r\n%s --> %s\r\n%s\r\n\r\n' % (str(pos), p.get('begin'), p.get('end'),
                                                          p.text.strip() if p.text is not None else '')
            subtitles[lang] = srt
        return subtitles

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        group_id = mobj.group('id')

        playlist = self._download_xml('http://www.bbc.co.uk/iplayer/playlist/%s' % group_id, group_id,
            'Downloading playlist XML')

        no_items = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}noItems')
        if no_items is not None:
            reason = no_items.get('reason')
            if reason == 'preAvailability':
                msg = 'Episode %s is not yet available' % group_id
            elif reason == 'postAvailability':
                msg = 'Episode %s is no longer available' % group_id
            else:
                msg = 'Episode %s is not available: %s' % (group_id, reason)
            raise ExtractorError(msg, expected=True)

        formats = []
        subtitles = None

        for item in self._extract_items(playlist):
            kind = item.get('kind')
            if kind != 'programme' and kind != 'radioProgramme':
                continue
            title = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}title').text
            description = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}summary').text

            programme_id = item.get('identifier')
            duration = int(item.get('duration'))

            media_selection = self._download_xml(
                'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/%s'  % programme_id,
                programme_id, 'Downloading media selection XML')

            for media in self._extract_medias(media_selection):
                kind = media.get('kind')
                if kind == 'audio':
                    formats.extend(self._extract_audio(media, programme_id))
                elif kind == 'video':
                    formats.extend(self._extract_video(media, programme_id))
                elif kind == 'captions':
                    subtitles = self._extract_captions(media, programme_id)

        if self._downloader.params.get('listsubtitles', False):
            self._list_available_subtitles(programme_id, subtitles)
            return

        self._sort_formats(formats)

        return {
            'id': programme_id,
            'title': title,
            'description': description,
            'duration': duration,
            'formats': formats,
            'subtitles': subtitles,
        }
Commit	Line	Data
082c6c86 S	1	from __future__ import unicode_literals
	2
	3	import re
	4
2e3fd9ec	5	from .subtitles import SubtitlesInfoExtractor
082c6c86 S	6	from ..utils import ExtractorError
	7
	8
2e3fd9ec	9	class BBCCoUkIE(SubtitlesInfoExtractor):
082c6c86	10	IE_NAME = 'bbc.co.uk'
2e3fd9ec	11	IE_DESC = 'BBC iPlayer'
8b7f7340	12	_VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:programmes\|iplayer/episode)/(?P<id>[\da-z]{8})'
082c6c86	13
2e3fd9ec S	14	_TESTS = [
	15	{
	16	'url': 'http://www.bbc.co.uk/programmes/p01q7wz1',
	17	'info_dict': {
	18	'id': 'p01q7wz4',
	19	'ext': 'flv',
	20	'title': 'Friction: Blu Mar Ten guest mix: Blu Mar Ten - Guest Mix',
	21	'description': 'Blu Mar Ten deliver a Guest Mix for Friction.',
	22	'duration': 1936,
	23	},
	24	'params': {
	25	# rtmp download
	26	'skip_download': True,
	27	}
082c6c86	28	},
2e3fd9ec S	29	{
	30	'url': 'http://www.bbc.co.uk/iplayer/episode/b00yng5w/The_Man_in_Black_Series_3_The_Printed_Name/',
	31	'info_dict': {
	32	'id': 'b00yng1d',
	33	'ext': 'flv',
	34	'title': 'The Man in Black: Series 3: The Printed Name',
	35	'description': "Mark Gatiss introduces Nicholas Pierpan's chilling tale of a writer's devilish pact with a mysterious man. Stars Ewan Bailey.",
	36	'duration': 1800,
	37	},
	38	'params': {
	39	# rtmp download
	40	'skip_download': True,
c7f0177f S	41	},
c7f0177f S	42	'skip': 'Episode is no longer available on BBC iPlayer Radio',
2e3fd9ec S	43	},
	44	{
	45	'url': 'http://www.bbc.co.uk/iplayer/episode/b03vhd1f/The_Voice_UK_Series_3_Blind_Auditions_5/',
	46	'info_dict': {
	47	'id': 'b00yng1d',
	48	'ext': 'flv',
17968e44 S	49	'title': 'The Voice UK: Series 3: Blind Auditions 5',
	50	'description': "Emma Willis and Marvin Humes present the fifth set of blind auditions in the singing competition, as the coaches continue to build their teams based on voice alone.",
	51	'duration': 5100,
2e3fd9ec S	52	},
	53	'params': {
	54	# rtmp download
	55	'skip_download': True,
	56	},
	57	'skip': 'Currently BBC iPlayer TV programmes are available to play in the UK only',
082c6c86	58	}
2e3fd9ec S	59	]
	60
	61	def _extract_asx_playlist(self, connection, programme_id):
	62	asx = self._download_xml(connection.get('href'), programme_id, 'Downloading ASX playlist')
	63	return [ref.get('href') for ref in asx.findall('./Entry/ref')]
	64
	65	def _extract_connection(self, connection, programme_id):
	66	formats = []
	67	protocol = connection.get('protocol')
	68	supplier = connection.get('supplier')
	69	if protocol == 'http':
	70	href = connection.get('href')
	71	# ASX playlist
	72	if supplier == 'asx':
	73	for i, ref in enumerate(self._extract_asx_playlist(connection, programme_id)):
	74	formats.append({
	75	'url': ref,
	76	'format_id': 'ref%s_%s' % (i, supplier),
	77	})
	78	# Direct link
	79	else:
	80	formats.append({
	81	'url': href,
	82	'format_id': supplier,
	83	})
	84	elif protocol == 'rtmp':
	85	application = connection.get('application', 'ondemand')
	86	auth_string = connection.get('authString')
	87	identifier = connection.get('identifier')
	88	server = connection.get('server')
	89	formats.append({
	90	'url': '%s://%s/%s?%s' % (protocol, server, application, auth_string),
	91	'play_path': identifier,
	92	'app': '%s?%s' % (application, auth_string),
	93	'page_url': 'http://www.bbc.co.uk',
	94	'player_url': 'http://www.bbc.co.uk/emp/releases/iplayer/revisions/617463_618125_4/617463_618125_4_emp.swf',
	95	'rtmp_live': False,
	96	'ext': 'flv',
	97	'format_id': supplier,
	98	})
	99	return formats
	100
	101	def _extract_items(self, playlist):
	102	return playlist.findall('./{http://bbc.co.uk/2008/emp/playlist}item')
	103
	104	def _extract_medias(self, media_selection):
	105	return media_selection.findall('./{http://bbc.co.uk/2008/mp/mediaselection}media')
	106
	107	def _extract_connections(self, media):
	108	return media.findall('./{http://bbc.co.uk/2008/mp/mediaselection}connection')
	109
	110	def _extract_video(self, media, programme_id):
	111	formats = []
	112	vbr = int(media.get('bitrate'))
	113	vcodec = media.get('encoding')
	114	service = media.get('service')
	115	width = int(media.get('width'))
	116	height = int(media.get('height'))
	117	file_size = int(media.get('media_file_size'))
	118	for connection in self._extract_connections(media):
	119	conn_formats = self._extract_connection(connection, programme_id)
	120	for format in conn_formats:
	121	format.update({
	122	'format_id': '%s_%s' % (service, format['format_id']),
123	'width': width,
124	'height': height,
125	'vbr': vbr,
126	'vcodec': vcodec,
127	'filesize': file_size,
128	})
129	formats.extend(conn_formats)
130	return formats
131
132	def _extract_audio(self, media, programme_id):
133	formats = []
134	abr = int(media.get('bitrate'))
135	acodec = media.get('encoding')
136	service = media.get('service')
137	for connection in self._extract_connections(media):
138	conn_formats = self._extract_connection(connection, programme_id)
139	for format in conn_formats:
140	format.update({
141	'format_id': '%s_%s' % (service, format['format_id']),
142	'abr': abr,
143	'acodec': acodec,
144	})
145	formats.extend(conn_formats)
146	return formats
147
148	def _extract_captions(self, media, programme_id):
149	subtitles = {}
150	for connection in self._extract_connections(media):
151	captions = self._download_xml(connection.get('href'), programme_id, 'Downloading captions')
152	lang = captions.get('{http://www.w3.org/XML/1998/namespace}lang', 'en')
153	ps = captions.findall('./{0}body/{0}div/{0}p'.format('{http://www.w3.org/2006/10/ttaf1}'))
154	srt = ''
155	for pos, p in enumerate(ps):
156	srt += '%s\r\n%s --> %s\r\n%s\r\n\r\n' % (str(pos), p.get('begin'), p.get('end'),
157	p.text.strip() if p.text is not None else '')
158	subtitles[lang] = srt
159	return subtitles
082c6c86 S	160
	161	def _real_extract(self, url):
	162	mobj = re.match(self._VALID_URL, url)
	163	group_id = mobj.group('id')
	164
	165	playlist = self._download_xml('http://www.bbc.co.uk/iplayer/playlist/%s' % group_id, group_id,
	166	'Downloading playlist XML')
	167
2e3fd9ec S	168	no_items = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}noItems')
	169	if no_items is not None:
	170	reason = no_items.get('reason')
	171	if reason == 'preAvailability':
	172	msg = 'Episode %s is not yet available' % group_id
	173	elif reason == 'postAvailability':
	174	msg = 'Episode %s is no longer available' % group_id
	175	else:
	176	msg = 'Episode %s is not available: %s' % (group_id, reason)
	177	raise ExtractorError(msg, expected=True)
082c6c86 S	178
082c6c86 S	179	formats = []
2e3fd9ec S	180	subtitles = None
	181
	182	for item in self._extract_items(playlist):
	183	kind = item.get('kind')
	184	if kind != 'programme' and kind != 'radioProgramme':
	185	continue
	186	title = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}title').text
	187	description = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}summary').text
	188
	189	programme_id = item.get('identifier')
	190	duration = int(item.get('duration'))
	191
	192	media_selection = self._download_xml(
	193	'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/%s' % programme_id,
	194	programme_id, 'Downloading media selection XML')
	195
	196	for media in self._extract_medias(media_selection):
	197	kind = media.get('kind')
	198	if kind == 'audio':
	199	formats.extend(self._extract_audio(media, programme_id))
	200	elif kind == 'video':
	201	formats.extend(self._extract_video(media, programme_id))
	202	elif kind == 'captions':
	203	subtitles = self._extract_captions(media, programme_id)
	204
	205	if self._downloader.params.get('listsubtitles', False):
	206	self._list_available_subtitles(programme_id, subtitles)
	207	return
082c6c86 S	208
	209	self._sort_formats(formats)
	210
	211	return {
2e3fd9ec	212	'id': programme_id,
082c6c86 S	213	'title': title,
	214	'description': description,
	215	'duration': duration,
	216	'formats': formats,
2e3fd9ec	217	'subtitles': subtitles,
082c6c86	218	}