[yt-dlp.git] / youtube_dl / extractor / limelight.py

# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    determine_ext,
    float_or_none,
    int_or_none,
)


class LimelightBaseIE(InfoExtractor):
    _PLAYLIST_SERVICE_URL = 'http://production-ps.lvp.llnw.net/r/PlaylistService/%s/%s/%s'
    _API_URL = 'http://api.video.limelight.com/rest/organizations/%s/%s/%s/%s.json'

    def _call_playlist_service(self, item_id, method, fatal=True):
        return self._download_json(
            self._PLAYLIST_SERVICE_URL % (self._PLAYLIST_SERVICE_PATH, item_id, method),
            item_id, 'Downloading PlaylistService %s JSON' % method, fatal=fatal)

    def _call_api(self, organization_id, item_id, method):
        return self._download_json(
            self._API_URL % (organization_id, self._API_PATH, item_id, method),
            item_id, 'Downloading API %s JSON' % method)

    def _extract(self, item_id, pc_method, mobile_method, meta_method):
        pc = self._call_playlist_service(item_id, pc_method)
        metadata = self._call_api(pc['orgId'], item_id, meta_method)
        mobile = self._call_playlist_service(item_id, mobile_method, fatal=False)
        return pc, mobile, metadata

    def _extract_info(self, streams, mobile_urls, properties):
        video_id = properties['media_id']
        formats = []
        urls = []
        for stream in streams:
            stream_url = stream.get('url')
            if not stream_url or stream.get('drmProtected') or stream_url in urls:
                continue
            urls.append(stream_url)
            ext = determine_ext(stream_url)
            if ext == 'f4m':
                formats.extend(self._extract_f4m_formats(
                    stream_url, video_id, f4m_id='hds', fatal=False))
            else:
                fmt = {
                    'url': stream_url,
                    'abr': float_or_none(stream.get('audioBitRate')),
                    'vbr': float_or_none(stream.get('videoBitRate')),
                    'fps': float_or_none(stream.get('videoFrameRate')),
                    'width': int_or_none(stream.get('videoWidthInPixels')),
                    'height': int_or_none(stream.get('videoHeightInPixels')),
                    'ext': ext,
                }
                rtmp = re.search(r'^(?P<url>rtmpe?://(?P<host>[^/]+)/(?P<app>.+))/(?P<playpath>mp4:.+)$', stream_url)
                if rtmp:
                    format_id = 'rtmp'
                    if stream.get('videoBitRate'):
                        format_id += '-%d' % int_or_none(stream['videoBitRate'])
                    http_url = 'http://cpl.delvenetworks.com/' + rtmp.group('playpath')[4:]
                    urls.append(http_url)
                    http_fmt = fmt.copy()
                    http_fmt.update({
                        'url': http_url,
                        'format_id': format_id.replace('rtmp', 'http'),
                    })
                    formats.append(http_fmt)
                    fmt.update({
                        'url': rtmp.group('url'),
                        'play_path': rtmp.group('playpath'),
                        'app': rtmp.group('app'),
                        'ext': 'flv',
                        'format_id': format_id,
                    })
                formats.append(fmt)

        for mobile_url in mobile_urls:
            media_url = mobile_url.get('mobileUrl')
            format_id = mobile_url.get('targetMediaPlatform')
            if not media_url or format_id in ('Widevine', 'SmoothStreaming') or media_url in urls:
                continue
            urls.append(media_url)
            ext = determine_ext(media_url)
            if ext == 'm3u8':
                formats.extend(self._extract_m3u8_formats(
                    media_url, video_id, 'mp4', 'm3u8_native',
                    m3u8_id=format_id, fatal=False))
            elif ext == 'f4m':
                formats.extend(self._extract_f4m_formats(
                    stream_url, video_id, f4m_id=format_id, fatal=False))
            else:
                formats.append({
                    'url': media_url,
                    'format_id': format_id,
                    'preference': -1,
                    'ext': ext,
                })

        self._sort_formats(formats)

        title = properties['title']
        description = properties.get('description')
        timestamp = int_or_none(properties.get('publish_date') or properties.get('create_date'))
        duration = float_or_none(properties.get('duration_in_milliseconds'), 1000)
        filesize = int_or_none(properties.get('total_storage_in_bytes'))
        categories = [properties.get('category')]
        tags = properties.get('tags', [])
        thumbnails = [{
            'url': thumbnail['url'],
            'width': int_or_none(thumbnail.get('width')),
            'height': int_or_none(thumbnail.get('height')),
        } for thumbnail in properties.get('thumbnails', []) if thumbnail.get('url')]

        subtitles = {}
        for caption in properties.get('captions', []):
            lang = caption.get('language_code')
            subtitles_url = caption.get('url')
            if lang and subtitles_url:
                subtitles.setdefault(lang, []).append({
                    'url': subtitles_url,
                })
        closed_captions_url = properties.get('closed_captions_url')
        if closed_captions_url:
            subtitles.setdefault('en', []).append({
                'url': closed_captions_url,
                'ext': 'ttml',
            })

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'formats': formats,
            'timestamp': timestamp,
            'duration': duration,
            'filesize': filesize,
            'categories': categories,
            'tags': tags,
            'thumbnails': thumbnails,
            'subtitles': subtitles,
        }


class LimelightMediaIE(LimelightBaseIE):
    IE_NAME = 'limelight'
    _VALID_URL = r'''(?x)
                        (?:
                            limelight:media:|
                            https?://
                                (?:
                                    link\.videoplatform\.limelight\.com/media/|
                                    assets\.delvenetworks\.com/player/loader\.swf
                                )
                                \?.*?\bmediaId=
                        )
                        (?P<id>[a-z0-9]{32})
                    '''
    _TESTS = [{
        'url': 'http://link.videoplatform.limelight.com/media/?mediaId=3ffd040b522b4485b6d84effc750cd86',
        'info_dict': {
            'id': '3ffd040b522b4485b6d84effc750cd86',
            'ext': 'mp4',
            'title': 'HaP and the HB Prince Trailer',
            'description': 'md5:8005b944181778e313d95c1237ddb640',
            'thumbnail': r're:^https?://.*\.jpeg$',
            'duration': 144.23,
            'timestamp': 1244136834,
            'upload_date': '20090604',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }, {
        # video with subtitles
        'url': 'limelight:media:a3e00274d4564ec4a9b29b9466432335',
        'md5': '2fa3bad9ac321e23860ca23bc2c69e3d',
        'info_dict': {
            'id': 'a3e00274d4564ec4a9b29b9466432335',
            'ext': 'mp4',
            'title': '3Play Media Overview Video',
            'thumbnail': r're:^https?://.*\.jpeg$',
            'duration': 78.101,
            'timestamp': 1338929955,
            'upload_date': '20120605',
            'subtitles': 'mincount:9',
        },
    }, {
        'url': 'https://assets.delvenetworks.com/player/loader.swf?mediaId=8018a574f08d416e95ceaccae4ba0452',
        'only_matching': True,
    }]
    _PLAYLIST_SERVICE_PATH = 'media'
    _API_PATH = 'media'

    def _real_extract(self, url):
        video_id = self._match_id(url)

        pc, mobile, metadata = self._extract(
            video_id, 'getPlaylistByMediaId', 'getMobilePlaylistByMediaId', 'properties')

        return self._extract_info(
            pc['playlistItems'][0].get('streams', []),
            mobile['mediaList'][0].get('mobileUrls', []) if mobile else [],
            metadata)


class LimelightChannelIE(LimelightBaseIE):
    IE_NAME = 'limelight:channel'
    _VALID_URL = r'''(?x)
                        (?:
                            limelight:channel:|
                            https?://
                                (?:
                                    link\.videoplatform\.limelight\.com/media/|
                                    assets\.delvenetworks\.com/player/loader\.swf
                                )
                                \?.*?\bchannelId=
                        )
                        (?P<id>[a-z0-9]{32})
                    '''
    _TESTS = [{
        'url': 'http://link.videoplatform.limelight.com/media/?channelId=ab6a524c379342f9b23642917020c082',
        'info_dict': {
            'id': 'ab6a524c379342f9b23642917020c082',
            'title': 'Javascript Sample Code',
        },
        'playlist_mincount': 3,
    }, {
        'url': 'http://assets.delvenetworks.com/player/loader.swf?channelId=ab6a524c379342f9b23642917020c082',
        'only_matching': True,
    }]
    _PLAYLIST_SERVICE_PATH = 'channel'
    _API_PATH = 'channels'

    def _real_extract(self, url):
        channel_id = self._match_id(url)

        pc, mobile, medias = self._extract(
            channel_id, 'getPlaylistByChannelId',
            'getMobilePlaylistWithNItemsByChannelId?begin=0&count=-1', 'media')

        entries = [
            self._extract_info(
                pc['playlistItems'][i].get('streams', []),
                mobile['mediaList'][i].get('mobileUrls', []) if mobile else [],
                medias['media_list'][i])
            for i in range(len(medias['media_list']))]

        return self.playlist_result(entries, channel_id, pc['title'])


class LimelightChannelListIE(LimelightBaseIE):
    IE_NAME = 'limelight:channel_list'
    _VALID_URL = r'''(?x)
                        (?:
                            limelight:channel_list:|
                            https?://
                                (?:
                                    link\.videoplatform\.limelight\.com/media/|
                                    assets\.delvenetworks\.com/player/loader\.swf
                                )
                                \?.*?\bchannelListId=
                        )
                        (?P<id>[a-z0-9]{32})
                    '''
    _TESTS = [{
        'url': 'http://link.videoplatform.limelight.com/media/?channelListId=301b117890c4465c8179ede21fd92e2b',
        'info_dict': {
            'id': '301b117890c4465c8179ede21fd92e2b',
            'title': 'Website - Hero Player',
        },
        'playlist_mincount': 2,
    }, {
        'url': 'https://assets.delvenetworks.com/player/loader.swf?channelListId=301b117890c4465c8179ede21fd92e2b',
        'only_matching': True,
    }]
    _PLAYLIST_SERVICE_PATH = 'channel_list'

    def _real_extract(self, url):
        channel_list_id = self._match_id(url)

        channel_list = self._call_playlist_service(channel_list_id, 'getMobileChannelListById')

        entries = [
            self.url_result('limelight:channel:%s' % channel['id'], 'LimelightChannel')
            for channel in channel_list['channelList']]

        return self.playlist_result(entries, channel_list_id, channel_list['title'])
Commit	Line	Data
ef5acfe3	1	# coding: utf-8
	2	from __future__ import unicode_literals
	3
	4	import re
	5
	6	from .common import InfoExtractor
	7	from ..utils import (
ef5acfe3	8	determine_ext,
d7fc5631 S	9	float_or_none,
d7fc5631 S	10	int_or_none,
ef5acfe3	11	)
	12
	13
d7fc5631 S	14	class LimelightBaseIE(InfoExtractor):
	15	_PLAYLIST_SERVICE_URL = 'http://production-ps.lvp.llnw.net/r/PlaylistService/%s/%s/%s'
	16	_API_URL = 'http://api.video.limelight.com/rest/organizations/%s/%s/%s/%s.json'
ef5acfe3	17
d7fc5631 S	18	def _call_playlist_service(self, item_id, method, fatal=True):
	19	return self._download_json(
	20	self._PLAYLIST_SERVICE_URL % (self._PLAYLIST_SERVICE_PATH, item_id, method),
	21	item_id, 'Downloading PlaylistService %s JSON' % method, fatal=fatal)
ef5acfe3	22
d7fc5631 S	23	def _call_api(self, organization_id, item_id, method):
	24	return self._download_json(
	25	self._API_URL % (organization_id, self._API_PATH, item_id, method),
	26	item_id, 'Downloading API %s JSON' % method)
ef5acfe3	27
d7fc5631 S	28	def _extract(self, item_id, pc_method, mobile_method, meta_method):
	29	pc = self._call_playlist_service(item_id, pc_method)
	30	metadata = self._call_api(pc['orgId'], item_id, meta_method)
	31	mobile = self._call_playlist_service(item_id, mobile_method, fatal=False)
	32	return pc, mobile, metadata
	33
	34	def _extract_info(self, streams, mobile_urls, properties):
ef5acfe3	35	video_id = properties['media_id']
ef5acfe3	36	formats = []
f8fd510e	37	urls = []
ef5acfe3	38	for stream in streams:
d7fc5631	39	stream_url = stream.get('url')
f8fd510e	40	if not stream_url or stream.get('drmProtected') or stream_url in urls:
d7fc5631	41	continue
f8fd510e	42	urls.append(stream_url)
e382b953 RA	43	ext = determine_ext(stream_url)
e382b953 RA	44	if ext == 'f4m':
8f1fddc8	45	formats.extend(self._extract_f4m_formats(
e382b953	46	stream_url, video_id, f4m_id='hds', fatal=False))
ef5acfe3	47	else:
ef5acfe3	48	fmt = {
d7fc5631 S	49	'url': stream_url,
	50	'abr': float_or_none(stream.get('audioBitRate')),
	51	'vbr': float_or_none(stream.get('videoBitRate')),
	52	'fps': float_or_none(stream.get('videoFrameRate')),
	53	'width': int_or_none(stream.get('videoWidthInPixels')),
	54	'height': int_or_none(stream.get('videoHeightInPixels')),
e382b953	55	'ext': ext,
ef5acfe3	56	}
42b7a5af	57	rtmp = re.search(r'^(?P<url>rtmpe?://(?P<host>[^/]+)/(?P<app>.+))/(?P<playpath>mp4:.+)$', stream_url)
ef5acfe3	58	if rtmp:
d7fc5631 S	59	format_id = 'rtmp'
	60	if stream.get('videoBitRate'):
	61	format_id += '-%d' % int_or_none(stream['videoBitRate'])
8bfda726	62	http_url = 'http://cpl.delvenetworks.com/' + rtmp.group('playpath')[4:]
f8fd510e	63	urls.append(http_url)
42b7a5af RA	64	http_fmt = fmt.copy()
42b7a5af RA	65	http_fmt.update({
f8fd510e	66	'url': http_url,
42b7a5af RA	67	'format_id': format_id.replace('rtmp', 'http'),
	68	})
	69	formats.append(http_fmt)
ef5acfe3	70	fmt.update({
	71	'url': rtmp.group('url'),
	72	'play_path': rtmp.group('playpath'),
	73	'app': rtmp.group('app'),
d7fc5631 S	74	'ext': 'flv',
d7fc5631 S	75	'format_id': format_id,
ef5acfe3	76	})
	77	formats.append(fmt)
	78
d7fc5631 S	79	for mobile_url in mobile_urls:
d7fc5631 S	80	media_url = mobile_url.get('mobileUrl')
d7fc5631	81	format_id = mobile_url.get('targetMediaPlatform')
f8fd510e	82	if not media_url or format_id in ('Widevine', 'SmoothStreaming') or media_url in urls:
e382b953	83	continue
f8fd510e	84	urls.append(media_url)
e382b953 RA	85	ext = determine_ext(media_url)
e382b953 RA	86	if ext == 'm3u8':
d7fc5631	87	formats.extend(self._extract_m3u8_formats(
8f1fddc8	88	media_url, video_id, 'mp4', 'm3u8_native',
8f1fddc8	89	m3u8_id=format_id, fatal=False))
e382b953 RA	90	elif ext == 'f4m':
	91	formats.extend(self._extract_f4m_formats(
	92	stream_url, video_id, f4m_id=format_id, fatal=False))
d7fc5631 S	93	else:
	94	formats.append({
	95	'url': media_url,
	96	'format_id': format_id,
	97	'preference': -1,
e382b953	98	'ext': ext,
d7fc5631 S	99	})
d7fc5631 S	100
ef5acfe3	101	self._sort_formats(formats)
	102
	103	title = properties['title']
	104	description = properties.get('description')
d7fc5631 S	105	timestamp = int_or_none(properties.get('publish_date') or properties.get('create_date'))
	106	duration = float_or_none(properties.get('duration_in_milliseconds'), 1000)
	107	filesize = int_or_none(properties.get('total_storage_in_bytes'))
ef5acfe3	108	categories = [properties.get('category')]
d7fc5631	109	tags = properties.get('tags', [])
ef5acfe3	110	thumbnails = [{
d7fc5631	111	'url': thumbnail['url'],
ef5acfe3	112	'width': int_or_none(thumbnail.get('width')),
ef5acfe3	113	'height': int_or_none(thumbnail.get('height')),
d7fc5631 S	114	} for thumbnail in properties.get('thumbnails', []) if thumbnail.get('url')]
	115
	116	subtitles = {}
21ac1a8a	117	for caption in properties.get('captions', []):
d7fc5631 S	118	lang = caption.get('language_code')
	119	subtitles_url = caption.get('url')
	120	if lang and subtitles_url:
fe458b65	121	subtitles.setdefault(lang, []).append({
d7fc5631	122	'url': subtitles_url,
fe458b65 S	123	})
	124	closed_captions_url = properties.get('closed_captions_url')
	125	if closed_captions_url:
	126	subtitles.setdefault('en', []).append({
	127	'url': closed_captions_url,
	128	'ext': 'ttml',
	129	})
ef5acfe3	130
	131	return {
	132	'id': video_id,
	133	'title': title,
	134	'description': description,
	135	'formats': formats,
	136	'timestamp': timestamp,
	137	'duration': duration,
	138	'filesize': filesize,
	139	'categories': categories,
d7fc5631	140	'tags': tags,
ef5acfe3	141	'thumbnails': thumbnails,
	142	'subtitles': subtitles,
	143	}
	144
	145
d7fc5631	146	class LimelightMediaIE(LimelightBaseIE):
ef5acfe3	147	IE_NAME = 'limelight'
79027c0e S	148	_VALID_URL = r'''(?x)
	149	(?:
	150	limelight:media:\|
	151	https?://
	152	(?:
	153	link\.videoplatform\.limelight\.com/media/\|
	154	assets\.delvenetworks\.com/player/loader\.swf
	155	)
	156	\?.*?\bmediaId=
	157	)
	158	(?P<id>[a-z0-9]{32})
	159	'''
9c544e25	160	_TESTS = [{
ef5acfe3	161	'url': 'http://link.videoplatform.limelight.com/media/?mediaId=3ffd040b522b4485b6d84effc750cd86',
ef5acfe3	162	'info_dict': {
ef5acfe3	163	'id': '3ffd040b522b4485b6d84effc750cd86',
e382b953	164	'ext': 'mp4',
ef5acfe3	165	'title': 'HaP and the HB Prince Trailer',
9c544e25	166	'description': 'md5:8005b944181778e313d95c1237ddb640',
ec85ded8	167	'thumbnail': r're:^https?://.*\.jpeg$',
d7fc5631	168	'duration': 144.23,
ef5acfe3	169	'timestamp': 1244136834,
d7fc5631 S	170	'upload_date': '20090604',
	171	},
	172	'params': {
e382b953	173	# m3u8 download
d7fc5631 S	174	'skip_download': True,
d7fc5631 S	175	},
9c544e25 S	176	}, {
	177	# video with subtitles
	178	'url': 'limelight:media:a3e00274d4564ec4a9b29b9466432335',
42b7a5af	179	'md5': '2fa3bad9ac321e23860ca23bc2c69e3d',
9c544e25 S	180	'info_dict': {
9c544e25 S	181	'id': 'a3e00274d4564ec4a9b29b9466432335',
42b7a5af	182	'ext': 'mp4',
9c544e25	183	'title': '3Play Media Overview Video',
ec85ded8	184	'thumbnail': r're:^https?://.*\.jpeg$',
9c544e25 S	185	'duration': 78.101,
	186	'timestamp': 1338929955,
	187	'upload_date': '20120605',
	188	'subtitles': 'mincount:9',
	189	},
79027c0e S	190	}, {
	191	'url': 'https://assets.delvenetworks.com/player/loader.swf?mediaId=8018a574f08d416e95ceaccae4ba0452',
	192	'only_matching': True,
9c544e25	193	}]
d7fc5631 S	194	_PLAYLIST_SERVICE_PATH = 'media'
d7fc5631 S	195	_API_PATH = 'media'
ef5acfe3	196
	197	def _real_extract(self, url):
	198	video_id = self._match_id(url)
	199
d7fc5631 S	200	pc, mobile, metadata = self._extract(
d7fc5631 S	201	video_id, 'getPlaylistByMediaId', 'getMobilePlaylistByMediaId', 'properties')
ef5acfe3	202
d7fc5631 S	203	return self._extract_info(
	204	pc['playlistItems'][0].get('streams', []),
	205	mobile['mediaList'][0].get('mobileUrls', []) if mobile else [],
	206	metadata)
ef5acfe3	207
ef5acfe3	208
d7fc5631	209	class LimelightChannelIE(LimelightBaseIE):
ef5acfe3	210	IE_NAME = 'limelight:channel'
79027c0e S	211	_VALID_URL = r'''(?x)
	212	(?:
	213	limelight:channel:\|
	214	https?://
	215	(?:
	216	link\.videoplatform\.limelight\.com/media/\|
	217	assets\.delvenetworks\.com/player/loader\.swf
	218	)
	219	\?.*?\bchannelId=
	220	)
	221	(?P<id>[a-z0-9]{32})
	222	'''
	223	_TESTS = [{
ef5acfe3	224	'url': 'http://link.videoplatform.limelight.com/media/?channelId=ab6a524c379342f9b23642917020c082',
	225	'info_dict': {
	226	'id': 'ab6a524c379342f9b23642917020c082',
	227	'title': 'Javascript Sample Code',
	228	},
	229	'playlist_mincount': 3,
79027c0e S	230	}, {
	231	'url': 'http://assets.delvenetworks.com/player/loader.swf?channelId=ab6a524c379342f9b23642917020c082',
	232	'only_matching': True,
	233	}]
d7fc5631 S	234	_PLAYLIST_SERVICE_PATH = 'channel'
d7fc5631 S	235	_API_PATH = 'channels'
ef5acfe3	236
	237	def _real_extract(self, url):
	238	channel_id = self._match_id(url)
	239
d7fc5631 S	240	pc, mobile, medias = self._extract(
	241	channel_id, 'getPlaylistByChannelId',
	242	'getMobilePlaylistWithNItemsByChannelId?begin=0&count=-1', 'media')
ef5acfe3	243
d7fc5631 S	244	entries = [
	245	self._extract_info(
	246	pc['playlistItems'][i].get('streams', []),
	247	mobile['mediaList'][i].get('mobileUrls', []) if mobile else [],
	248	medias['media_list'][i])
	249	for i in range(len(medias['media_list']))]
ef5acfe3	250
d7fc5631	251	return self.playlist_result(entries, channel_id, pc['title'])
ef5acfe3	252
ef5acfe3	253
d7fc5631	254	class LimelightChannelListIE(LimelightBaseIE):
ef5acfe3	255	IE_NAME = 'limelight:channel_list'
79027c0e S	256	_VALID_URL = r'''(?x)
	257	(?:
	258	limelight:channel_list:\|
	259	https?://
	260	(?:
	261	link\.videoplatform\.limelight\.com/media/\|
	262	assets\.delvenetworks\.com/player/loader\.swf
	263	)
	264	\?.*?\bchannelListId=
	265	)
	266	(?P<id>[a-z0-9]{32})
	267	'''
	268	_TESTS = [{
ef5acfe3	269	'url': 'http://link.videoplatform.limelight.com/media/?channelListId=301b117890c4465c8179ede21fd92e2b',
	270	'info_dict': {
	271	'id': '301b117890c4465c8179ede21fd92e2b',
	272	'title': 'Website - Hero Player',
	273	},
	274	'playlist_mincount': 2,
79027c0e S	275	}, {
	276	'url': 'https://assets.delvenetworks.com/player/loader.swf?channelListId=301b117890c4465c8179ede21fd92e2b',
	277	'only_matching': True,
	278	}]
d7fc5631	279	_PLAYLIST_SERVICE_PATH = 'channel_list'
ef5acfe3	280
	281	def _real_extract(self, url):
	282	channel_list_id = self._match_id(url)
	283
d7fc5631	284	channel_list = self._call_playlist_service(channel_list_id, 'getMobileChannelListById')
ef5acfe3	285
d7fc5631 S	286	entries = [
	287	self.url_result('limelight:channel:%s' % channel['id'], 'LimelightChannel')
	288	for channel in channel_list['channelList']]
ef5acfe3	289
d7fc5631	290	return self.playlist_result(entries, channel_list_id, channel_list['title'])