[yt-dlp.git] / youtube_dl / extractor / letv.py

# coding: utf-8
from __future__ import unicode_literals

import datetime
import re
import time
import base64

from .common import InfoExtractor
from ..compat import (
    compat_urllib_parse,
    compat_ord,
)
from ..utils import (
    determine_ext,
    ExtractorError,
    parse_iso8601,
    sanitized_Request,
    int_or_none,
    str_or_none,
    encode_data_uri,
)


class LetvIE(InfoExtractor):
    IE_DESC = '乐视网'
    _VALID_URL = r'http://www\.letv\.com/ptv/vplay/(?P<id>\d+).html'

    _TESTS = [{
        'url': 'http://www.letv.com/ptv/vplay/22005890.html',
        'md5': 'edadcfe5406976f42f9f266057ee5e40',
        'info_dict': {
            'id': '22005890',
            'ext': 'mp4',
            'title': '第87届奥斯卡颁奖礼完美落幕 《鸟人》成最大赢家',
            'description': 'md5:a9cb175fd753e2962176b7beca21a47c',
        },
        'params': {
            'hls_prefer_native': True,
        },
    }, {
        'url': 'http://www.letv.com/ptv/vplay/1415246.html',
        'info_dict': {
            'id': '1415246',
            'ext': 'mp4',
            'title': '美人天下01',
            'description': 'md5:f88573d9d7225ada1359eaf0dbf8bcda',
        },
        'params': {
            'hls_prefer_native': True,
        },
    }, {
        'note': 'This video is available only in Mainland China, thus a proxy is needed',
        'url': 'http://www.letv.com/ptv/vplay/1118082.html',
        'md5': '2424c74948a62e5f31988438979c5ad1',
        'info_dict': {
            'id': '1118082',
            'ext': 'mp4',
            'title': '与龙共舞 完整版',
            'description': 'md5:7506a5eeb1722bb9d4068f85024e3986',
        },
        'params': {
            'hls_prefer_native': True,
        },
        'skip': 'Only available in China',
    }]

    @staticmethod
    def urshift(val, n):
        return val >> n if val >= 0 else (val + 0x100000000) >> n

    # ror() and calc_time_key() are reversed from a embedded swf file in KLetvPlayer.swf
    def ror(self, param1, param2):
        _loc3_ = 0
        while _loc3_ < param2:
            param1 = self.urshift(param1, 1) + ((param1 & 1) << 31)
            _loc3_ += 1
        return param1

    def calc_time_key(self, param1):
        _loc2_ = 773625421
        _loc3_ = self.ror(param1, _loc2_ % 13)
        _loc3_ = _loc3_ ^ _loc2_
        _loc3_ = self.ror(_loc3_, _loc2_ % 17)
        return _loc3_

    # see M3U8Encryption class in KLetvPlayer.swf
    @staticmethod
    def decrypt_m3u8(encrypted_data):
        if encrypted_data[:5].decode('utf-8').lower() != 'vc_01':
            return encrypted_data
        encrypted_data = encrypted_data[5:]

        _loc4_ = bytearray()
        while encrypted_data:
            b = compat_ord(encrypted_data[0])
            _loc4_.extend([b // 16, b & 0x0f])
            encrypted_data = encrypted_data[1:]
        idx = len(_loc4_) - 11
        _loc4_ = _loc4_[idx:] + _loc4_[:idx]
        _loc7_ = bytearray()
        while _loc4_:
            _loc7_.append(_loc4_[0] * 16 + _loc4_[1])
            _loc4_ = _loc4_[2:]

        return bytes(_loc7_)

    def _real_extract(self, url):
        media_id = self._match_id(url)
        page = self._download_webpage(url, media_id)
        params = {
            'id': media_id,
            'platid': 1,
            'splatid': 101,
            'format': 1,
            'tkey': self.calc_time_key(int(time.time())),
            'domain': 'www.letv.com'
        }
        play_json_req = sanitized_Request(
            'http://api.letv.com/mms/out/video/playJson?' + compat_urllib_parse.urlencode(params)
        )
        cn_verification_proxy = self._downloader.params.get('cn_verification_proxy')
        if cn_verification_proxy:
            play_json_req.add_header('Ytdl-request-proxy', cn_verification_proxy)

        play_json = self._download_json(
            play_json_req,
            media_id, 'Downloading playJson data')

        # Check for errors
        playstatus = play_json['playstatus']
        if playstatus['status'] == 0:
            flag = playstatus['flag']
            if flag == 1:
                msg = 'Country %s auth error' % playstatus['country']
            else:
                msg = 'Generic error. flag = %d' % flag
            raise ExtractorError(msg, expected=True)

        playurl = play_json['playurl']

        formats = ['350', '1000', '1300', '720p', '1080p']
        dispatch = playurl['dispatch']

        urls = []
        for format_id in formats:
            if format_id in dispatch:
                media_url = playurl['domain'][0] + dispatch[format_id][0]
                media_url += '&' + compat_urllib_parse.urlencode({
                    'm3v': 1,
                    'format': 1,
                    'expect': 3,
                    'rateid': format_id,
                })

                nodes_data = self._download_json(
                    media_url, media_id,
                    'Download JSON metadata for format %s' % format_id)

                req = self._request_webpage(
                    nodes_data['nodelist'][0]['location'], media_id,
                    note='Downloading m3u8 information for format %s' % format_id)

                m3u8_data = self.decrypt_m3u8(req.read())

                url_info_dict = {
                    'url': encode_data_uri(m3u8_data, 'application/vnd.apple.mpegurl'),
                    'ext': determine_ext(dispatch[format_id][1]),
                    'format_id': format_id,
                    'protocol': 'm3u8',
                }

                if format_id[-1:] == 'p':
                    url_info_dict['height'] = int_or_none(format_id[:-1])

                urls.append(url_info_dict)

        publish_time = parse_iso8601(self._html_search_regex(
            r'发布时间&nbsp;([^<>]+) ', page, 'publish time', default=None),
            delimiter=' ', timezone=datetime.timedelta(hours=8))
        description = self._html_search_meta('description', page, fatal=False)

        return {
            'id': media_id,
            'formats': urls,
            'title': playurl['title'],
            'thumbnail': playurl['pic'],
            'description': description,
            'timestamp': publish_time,
        }


class LetvTvIE(InfoExtractor):
    _VALID_URL = r'http://www.letv.com/tv/(?P<id>\d+).html'
    _TESTS = [{
        'url': 'http://www.letv.com/tv/46177.html',
        'info_dict': {
            'id': '46177',
            'title': '美人天下',
            'description': 'md5:395666ff41b44080396e59570dbac01c'
        },
        'playlist_count': 35
    }]

    def _real_extract(self, url):
        playlist_id = self._match_id(url)
        page = self._download_webpage(url, playlist_id)

        media_urls = list(set(re.findall(
            r'http://www.letv.com/ptv/vplay/\d+.html', page)))
        entries = [self.url_result(media_url, ie='Letv')
                   for media_url in media_urls]

        title = self._html_search_meta('keywords', page,
                                       fatal=False).split('，')[0]
        description = self._html_search_meta('description', page, fatal=False)

        return self.playlist_result(entries, playlist_id, playlist_title=title,
                                    playlist_description=description)


class LetvPlaylistIE(LetvTvIE):
    _VALID_URL = r'http://tv.letv.com/[a-z]+/(?P<id>[a-z]+)/index.s?html'
    _TESTS = [{
        'url': 'http://tv.letv.com/izt/wuzetian/index.html',
        'info_dict': {
            'id': 'wuzetian',
            'title': '武媚娘传奇',
            'description': 'md5:e12499475ab3d50219e5bba00b3cb248'
        },
        # This playlist contains some extra videos other than the drama itself
        'playlist_mincount': 96
    }, {
        'url': 'http://tv.letv.com/pzt/lswjzzjc/index.shtml',
        'info_dict': {
            'id': 'lswjzzjc',
            # The title should be "劲舞青春", but I can't find a simple way to
            # determine the playlist title
            'title': '乐视午间自制剧场',
            'description': 'md5:b1eef244f45589a7b5b1af9ff25a4489'
        },
        'playlist_mincount': 7
    }]


class LetvCloudIE(InfoExtractor):
    IE_DESC = '乐视云'
    _VALID_URL = r'http://yuntv\.letv\.com/bcloud.html\?.*$'

    _TESTS = [{
        'url': 'http://yuntv.letv.com/bcloud.html?uu=p7jnfw5hw9&vu=467623dedf',
        'md5': '26450599afd64c513bc77030ad15db44',
        'info_dict': {
            'id': 'p7jnfw5hw9_467623dedf',
            'ext': 'mp4',
            'title': 'p7jnfw5hw9_467623dedf',
        },
    }, {
        'url': 'http://yuntv.letv.com/bcloud.html?uu=p7jnfw5hw9&vu=ec93197892&pu=2c7cd40209&auto_play=1&gpcflag=1&width=640&height=360',
        'info_dict': {
            'id': 'p7jnfw5hw9_ec93197892',
            'ext': 'mp4',
            'title': 'p7jnfw5hw9_ec93197892',
        },
    }, {
        'url': 'http://yuntv.letv.com/bcloud.html?uu=p7jnfw5hw9&vu=187060b6fd',
        'info_dict': {
            'id': 'p7jnfw5hw9_187060b6fd',
            'ext': 'mp4',
            'title': 'p7jnfw5hw9_187060b6fd',
        },
    }]

    def _real_extract(self, url):
        uu_mobj = re.search('uu=([\w]+)', url)
        vu_mobj = re.search('vu=([\w]+)', url)

        if not uu_mobj or not vu_mobj:
            raise ExtractorError('Invalid URL: %s' % url, expected=True)

        uu = uu_mobj.group(1)
        vu = vu_mobj.group(1)
        media_id = uu + '_' + vu

        play_json_req = sanitized_Request(
            'http://api.letvcloud.com/gpc.php?cf=html5&sign=signxxxxx&ver=2.2&format=json&' +
            'uu=' + uu + '&vu=' + vu)
        play_json = self._download_json(play_json_req, media_id, 'Downloading playJson data')

        formats = []
        for media in play_json['data']['video_info']['media'].values():
            play_url = media['play_url']
            formats.append({
                'url': base64.b64decode(play_url['main_url'].encode('utf-8')).decode('utf-8'),
                'ext': 'mp4',
                'format_id': int_or_none(play_url.get('vtype')),
                'format_note': str_or_none(play_url.get('definition')),
                'width': int_or_none(play_url.get('vwidth')),
                'height': int_or_none(play_url.get('vheight')),
            })
        self._sort_formats(formats)

        return {
            'id': media_id,
            'title': media_id,
            'formats': formats,
        }
Commit	Line	Data
7f09a662 YCH	1	# coding: utf-8
	2	from __future__ import unicode_literals
	3
265bfa2c	4	import datetime
57031161	5	import re
7f09a662	6	import time
5f432ac8	7	import base64
7f09a662 YCH	8
7f09a662 YCH	9	from .common import InfoExtractor
265bfa2c	10	from ..compat import (
265bfa2c	11	compat_urllib_parse,
1e399778	12	compat_ord,
265bfa2c PH	13	)
	14	from ..utils import (
	15	determine_ext,
	16	ExtractorError,
	17	parse_iso8601,
5c2266df	18	sanitized_Request,
593ddd85	19	int_or_none,
5f432ac8	20	str_or_none,
1e399778	21	encode_data_uri,
265bfa2c	22	)
7f09a662 YCH	23
	24
	25	class LetvIE(InfoExtractor):
963d0ce7	26	IE_DESC = '乐视网'
57031161	27	_VALID_URL = r'http://www\.letv\.com/ptv/vplay/(?P<id>\d+).html'
7f09a662 YCH	28
	29	_TESTS = [{
	30	'url': 'http://www.letv.com/ptv/vplay/22005890.html',
1e399778	31	'md5': 'edadcfe5406976f42f9f266057ee5e40',
7f09a662 YCH	32	'info_dict': {
	33	'id': '22005890',
	34	'ext': 'mp4',
	35	'title': '第87届奥斯卡颁奖礼完美落幕《鸟人》成最大赢家',
67706359	36	'description': 'md5:a9cb175fd753e2962176b7beca21a47c',
1e399778 YCH	37	},
	38	'params': {
	39	'hls_prefer_native': True,
	40	},
7f09a662	41	}, {
67706359	42	'url': 'http://www.letv.com/ptv/vplay/1415246.html',
7f09a662	43	'info_dict': {
67706359	44	'id': '1415246',
7f09a662	45	'ext': 'mp4',
67706359 YCH	46	'title': '美人天下01',
	47	'description': 'md5:f88573d9d7225ada1359eaf0dbf8bcda',
	48	},
1e399778 YCH	49	'params': {
	50	'hls_prefer_native': True,
	51	},
91410c9b PH	52	}, {
	53	'note': 'This video is available only in Mainland China, thus a proxy is needed',
	54	'url': 'http://www.letv.com/ptv/vplay/1118082.html',
1e399778	55	'md5': '2424c74948a62e5f31988438979c5ad1',
91410c9b PH	56	'info_dict': {
	57	'id': '1118082',
	58	'ext': 'mp4',
	59	'title': '与龙共舞完整版',
	60	'description': 'md5:7506a5eeb1722bb9d4068f85024e3986',
	61	},
1e399778 YCH	62	'params': {
	63	'hls_prefer_native': True,
	64	},
051df9ad	65	'skip': 'Only available in China',
7f09a662 YCH	66	}]
	67
	68	@staticmethod
	69	def urshift(val, n):
	70	return val >> n if val >= 0 else (val + 0x100000000) >> n
	71
265bfa2c	72	# ror() and calc_time_key() are reversed from a embedded swf file in KLetvPlayer.swf
7f09a662 YCH	73	def ror(self, param1, param2):
	74	_loc3_ = 0
	75	while _loc3_ < param2:
	76	param1 = self.urshift(param1, 1) + ((param1 & 1) << 31)
	77	_loc3_ += 1
	78	return param1
	79
265bfa2c	80	def calc_time_key(self, param1):
7f09a662 YCH	81	_loc2_ = 773625421
	82	_loc3_ = self.ror(param1, _loc2_ % 13)
	83	_loc3_ = _loc3_ ^ _loc2_
	84	_loc3_ = self.ror(_loc3_, _loc2_ % 17)
	85	return _loc3_
	86
1e399778 YCH	87	# see M3U8Encryption class in KLetvPlayer.swf
	88	@staticmethod
	89	def decrypt_m3u8(encrypted_data):
	90	if encrypted_data[:5].decode('utf-8').lower() != 'vc_01':
	91	return encrypted_data
	92	encrypted_data = encrypted_data[5:]
	93
	94	_loc4_ = bytearray()
	95	while encrypted_data:
	96	b = compat_ord(encrypted_data[0])
	97	_loc4_.extend([b // 16, b & 0x0f])
	98	encrypted_data = encrypted_data[1:]
	99	idx = len(_loc4_) - 11
	100	_loc4_ = _loc4_[idx:] + _loc4_[:idx]
	101	_loc7_ = bytearray()
	102	while _loc4_:
	103	_loc7_.append(_loc4_[0] * 16 + _loc4_[1])
	104	_loc4_ = _loc4_[2:]
	105
	106	return bytes(_loc7_)
	107
7f09a662 YCH	108	def _real_extract(self, url):
	109	media_id = self._match_id(url)
	110	page = self._download_webpage(url, media_id)
	111	params = {
	112	'id': media_id,
	113	'platid': 1,
	114	'splatid': 101,
	115	'format': 1,
265bfa2c	116	'tkey': self.calc_time_key(int(time.time())),
7f09a662 YCH	117	'domain': 'www.letv.com'
7f09a662 YCH	118	}
5c2266df	119	play_json_req = sanitized_Request(
91410c9b PH	120	'http://api.letv.com/mms/out/video/playJson?' + compat_urllib_parse.urlencode(params)
91410c9b PH	121	)
63fc8000 YCH	122	cn_verification_proxy = self._downloader.params.get('cn_verification_proxy')
	123	if cn_verification_proxy:
	124	play_json_req.add_header('Ytdl-request-proxy', cn_verification_proxy)
	125
7f09a662	126	play_json = self._download_json(
91410c9b	127	play_json_req,
576904bc	128	media_id, 'Downloading playJson data')
7f09a662 YCH	129
	130	# Check for errors
	131	playstatus = play_json['playstatus']
	132	if playstatus['status'] == 0:
	133	flag = playstatus['flag']
	134	if flag == 1:
	135	msg = 'Country %s auth error' % playstatus['country']
	136	else:
	137	msg = 'Generic error. flag = %d' % flag
	138	raise ExtractorError(msg, expected=True)
	139
	140	playurl = play_json['playurl']
	141
	142	formats = ['350', '1000', '1300', '720p', '1080p']
	143	dispatch = playurl['dispatch']
	144
	145	urls = []
	146	for format_id in formats:
	147	if format_id in dispatch:
	148	media_url = playurl['domain'][0] + dispatch[format_id][0]
1e399778 YCH	149	media_url += '&' + compat_urllib_parse.urlencode({
	150	'm3v': 1,
	151	'format': 1,
	152	'expect': 3,
	153	'rateid': format_id,
7f09a662	154	})
1e399778 YCH	155
	156	nodes_data = self._download_json(
	157	media_url, media_id,
	158	'Download JSON metadata for format %s' % format_id)
	159
	160	req = self._request_webpage(
	161	nodes_data['nodelist'][0]['location'], media_id,
	162	note='Downloading m3u8 information for format %s' % format_id)
	163
	164	m3u8_data = self.decrypt_m3u8(req.read())
7f09a662 YCH	165
7f09a662 YCH	166	url_info_dict = {
05a3879f	167	'url': encode_data_uri(m3u8_data, 'application/vnd.apple.mpegurl'),
91410c9b PH	168	'ext': determine_ext(dispatch[format_id][1]),
91410c9b PH	169	'format_id': format_id,
1e399778	170	'protocol': 'm3u8',
7f09a662 YCH	171	}
	172
	173	if format_id[-1:] == 'p':
593ddd85	174	url_info_dict['height'] = int_or_none(format_id[:-1])
7f09a662 YCH	175
	176	urls.append(url_info_dict)
	177
	178	publish_time = parse_iso8601(self._html_search_regex(
91410c9b	179	r'发布时间 ([^<>]+) ', page, 'publish time', default=None),
7f09a662	180	delimiter=' ', timezone=datetime.timedelta(hours=8))
67706359	181	description = self._html_search_meta('description', page, fatal=False)
7f09a662 YCH	182
	183	return {
	184	'id': media_id,
	185	'formats': urls,
	186	'title': playurl['title'],
	187	'thumbnail': playurl['pic'],
67706359	188	'description': description,
7f09a662 YCH	189	'timestamp': publish_time,
7f09a662 YCH	190	}
57031161 YCH	191
	192
	193	class LetvTvIE(InfoExtractor):
	194	_VALID_URL = r'http://www.letv.com/tv/(?P<id>\d+).html'
	195	_TESTS = [{
	196	'url': 'http://www.letv.com/tv/46177.html',
	197	'info_dict': {
	198	'id': '46177',
	199	'title': '美人天下',
	200	'description': 'md5:395666ff41b44080396e59570dbac01c'
	201	},
	202	'playlist_count': 35
	203	}]
	204
	205	def _real_extract(self, url):
	206	playlist_id = self._match_id(url)
	207	page = self._download_webpage(url, playlist_id)
	208
	209	media_urls = list(set(re.findall(
	210	r'http://www.letv.com/ptv/vplay/\d+.html', page)))
	211	entries = [self.url_result(media_url, ie='Letv')
	212	for media_url in media_urls]
	213
67706359 YCH	214	title = self._html_search_meta('keywords', page,
67706359 YCH	215	fatal=False).split('，')[0]
57031161 YCH	216	description = self._html_search_meta('description', page, fatal=False)
	217
	218	return self.playlist_result(entries, playlist_id, playlist_title=title,
	219	playlist_description=description)
	220
	221
	222	class LetvPlaylistIE(LetvTvIE):
	223	_VALID_URL = r'http://tv.letv.com/[a-z]+/(?P<id>[a-z]+)/index.s?html'
	224	_TESTS = [{
	225	'url': 'http://tv.letv.com/izt/wuzetian/index.html',
	226	'info_dict': {
	227	'id': 'wuzetian',
	228	'title': '武媚娘传奇',
	229	'description': 'md5:e12499475ab3d50219e5bba00b3cb248'
	230	},
67706359 YCH	231	# This playlist contains some extra videos other than the drama itself
67706359 YCH	232	'playlist_mincount': 96
57031161 YCH	233	}, {
	234	'url': 'http://tv.letv.com/pzt/lswjzzjc/index.shtml',
	235	'info_dict': {
	236	'id': 'lswjzzjc',
67706359 YCH	237	# The title should be "劲舞青春", but I can't find a simple way to
67706359 YCH	238	# determine the playlist title
57031161 YCH	239	'title': '乐视午间自制剧场',
	240	'description': 'md5:b1eef244f45589a7b5b1af9ff25a4489'
	241	},
	242	'playlist_mincount': 7
	243	}]
5f432ac8 FF	244
	245
	246	class LetvCloudIE(InfoExtractor):
	247	IE_DESC = '乐视云'
	248	_VALID_URL = r'http://yuntv\.letv\.com/bcloud.html\?.*$'
	249
	250	_TESTS = [{
	251	'url': 'http://yuntv.letv.com/bcloud.html?uu=p7jnfw5hw9&vu=467623dedf',
	252	'md5': '26450599afd64c513bc77030ad15db44',
	253	'info_dict': {
	254	'id': 'p7jnfw5hw9_467623dedf',
	255	'ext': 'mp4',
	256	'title': 'p7jnfw5hw9_467623dedf',
	257	},
	258	}, {
	259	'url': 'http://yuntv.letv.com/bcloud.html?uu=p7jnfw5hw9&vu=ec93197892&pu=2c7cd40209&auto_play=1&gpcflag=1&width=640&height=360',
	260	'info_dict': {
	261	'id': 'p7jnfw5hw9_ec93197892',
	262	'ext': 'mp4',
	263	'title': 'p7jnfw5hw9_ec93197892',
	264	},
	265	}, {
	266	'url': 'http://yuntv.letv.com/bcloud.html?uu=p7jnfw5hw9&vu=187060b6fd',
	267	'info_dict': {
	268	'id': 'p7jnfw5hw9_187060b6fd',
	269	'ext': 'mp4',
	270	'title': 'p7jnfw5hw9_187060b6fd',
	271	},
	272	}]
	273
	274	def _real_extract(self, url):
e0690782 YCH	275	uu_mobj = re.search('uu=([\w]+)', url)
	276	vu_mobj = re.search('vu=([\w]+)', url)
	277
	278	if not uu_mobj or not vu_mobj:
	279	raise ExtractorError('Invalid URL: %s' % url, expected=True)
	280
	281	uu = uu_mobj.group(1)
	282	vu = vu_mobj.group(1)
5f432ac8 FF	283	media_id = uu + '_' + vu
	284
	285	play_json_req = sanitized_Request(
	286	'http://api.letvcloud.com/gpc.php?cf=html5&sign=signxxxxx&ver=2.2&format=json&' +
8fff4f61	287	'uu=' + uu + '&vu=' + vu)
5f432ac8 FF	288	play_json = self._download_json(play_json_req, media_id, 'Downloading playJson data')
5f432ac8 FF	289
10defdd0 YCH	290	formats = []
	291	for media in play_json['data']['video_info']['media'].values():
	292	play_url = media['play_url']
	293	formats.append({
8fff4f61	294	'url': base64.b64decode(play_url['main_url'].encode('utf-8')).decode('utf-8'),
10defdd0 YCH	295	'ext': 'mp4',
	296	'format_id': int_or_none(play_url.get('vtype')),
	297	'format_note': str_or_none(play_url.get('definition')),
	298	'width': int_or_none(play_url.get('vwidth')),
	299	'height': int_or_none(play_url.get('vheight')),
	300	})
5f432ac8 FF	301	self._sort_formats(formats)
	302
	303	return {
	304	'id': media_id,
	305	'title': media_id,
	306	'formats': formats,
	307	}