[yt-dlp.git] / youtube_dl / extractor / viki.py

# coding: utf-8
from __future__ import unicode_literals

import json
import time
import hmac
import hashlib
import itertools

from ..utils import (
    ExtractorError,
    int_or_none,
    parse_age_limit,
    parse_iso8601,
)
from ..compat import compat_urllib_request
from .common import InfoExtractor


class VikiBaseIE(InfoExtractor):
    _VALID_URL_BASE = r'https?://(?:www\.)?viki\.(?:com|net|mx|jp|fr)/'
    _API_QUERY_TEMPLATE = '/v4/%sapp=%s&t=%s&site=www.viki.com'
    _API_URL_TEMPLATE = 'http://api.viki.io%s&sig=%s'

    _APP = '65535a'
    _APP_VERSION = '2.2.5.1428709186'
    _APP_SECRET = '-$iJ}@p7!G@SyU/je1bEyWg}upLu-6V6-Lg9VD(]siH,r.,m-r|ulZ,U4LC/SeR)'

    _NETRC_MACHINE = 'viki'

    def _prepare_call(self, path, timestamp=None, post_data=None):
        path += '?' if '?' not in path else '&'
        if not timestamp:
            timestamp = int(time.time())
        query = self._API_QUERY_TEMPLATE % (path, self._APP, timestamp)
        sig = hmac.new(
            self._APP_SECRET.encode('ascii'),
            query.encode('ascii'),
            hashlib.sha1
        ).hexdigest()
        url = self._API_URL_TEMPLATE % (query, sig)
        return compat_urllib_request.Request(
            url, json.dumps(post_data).encode('utf-8')) if post_data else url

    def _call_api(self, path, video_id, note, timestamp=None, post_data=None):
        resp = self._download_json(
            self._prepare_call(path, timestamp, post_data), video_id, note)

        error = resp.get('error')
        if error:
            if error == 'invalid timestamp':
                resp = self._download_json(
                    self._prepare_call(path, int(resp['current_timestamp']), post_data),
                    video_id, '%s (retry)' % note)
                error = resp.get('error')
            if error:
                self._raise_error(resp['error'])

        return resp

    def _raise_error(self, error):
        raise ExtractorError(
            '%s returned error: %s' % (self.IE_NAME, error),
            expected=True)

    def _real_initialize(self):
        self._login()

    def _login(self):
        (username, password) = self._get_login_info()
        if username is None:
            return

        login_form = {
            'login_id': username,
            'password': password,
        }

        self._call_api(
            'sessions.json', None,
            'Logging in as %s' % username, post_data=login_form)


class VikiIE(VikiBaseIE):
    IE_NAME = 'viki'
    _VALID_URL = r'%s(?:videos|player)/(?P<id>[0-9]+v)' % VikiBaseIE._VALID_URL_BASE
    _TESTS = [{
        'url': 'http://www.viki.com/videos/1023585v-heirs-episode-14',
        'info_dict': {
            'id': '1023585v',
            'ext': 'mp4',
            'title': 'Heirs Episode 14',
            'uploader': 'SBS',
            'description': 'md5:c4b17b9626dd4b143dcc4d855ba3474e',
            'upload_date': '20131121',
            'age_limit': 13,
        },
        'skip': 'Blocked in the US',
    }, {
        # clip
        'url': 'http://www.viki.com/videos/1067139v-the-avengers-age-of-ultron-press-conference',
        'md5': '86c0b5dbd4d83a6611a79987cc7a1989',
        'info_dict': {
            'id': '1067139v',
            'ext': 'mp4',
            'title': "'The Avengers: Age of Ultron' Press Conference",
            'description': 'md5:d70b2f9428f5488321bfe1db10d612ea',
            'duration': 352,
            'timestamp': 1430380829,
            'upload_date': '20150430',
            'uploader': 'Arirang TV',
            'like_count': int,
            'age_limit': 0,
        }
    }, {
        'url': 'http://www.viki.com/videos/1048879v-ankhon-dekhi',
        'info_dict': {
            'id': '1048879v',
            'ext': 'mp4',
            'title': 'Ankhon Dekhi',
            'duration': 6512,
            'timestamp': 1408532356,
            'upload_date': '20140820',
            'uploader': 'Spuul',
            'like_count': int,
            'age_limit': 13,
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        }
    }, {
        # episode
        'url': 'http://www.viki.com/videos/44699v-boys-over-flowers-episode-1',
        'md5': '190f3ef426005ba3a080a63325955bc3',
        'info_dict': {
            'id': '44699v',
            'ext': 'mp4',
            'title': 'Boys Over Flowers - Episode 1',
            'description': 'md5:52617e4f729c7d03bfd4bcbbb6e946f2',
            'duration': 4155,
            'timestamp': 1270496524,
            'upload_date': '20100405',
            'uploader': 'group8',
            'like_count': int,
            'age_limit': 13,
        }
    }, {
        # youtube external
        'url': 'http://www.viki.com/videos/50562v-poor-nastya-complete-episode-1',
        'md5': '216d1afdc0c64d1febc1e9f2bd4b864b',
        'info_dict': {
            'id': '50562v',
            'ext': 'mp4',
            'title': 'Poor Nastya [COMPLETE] - Episode 1',
            'description': '',
            'duration': 607,
            'timestamp': 1274949505,
            'upload_date': '20101213',
            'uploader': 'ad14065n',
            'uploader_id': 'ad14065n',
            'like_count': int,
            'age_limit': 13,
        }
    }, {
        'url': 'http://www.viki.com/player/44699v',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        video = self._call_api(
            'videos/%s.json' % video_id, video_id, 'Downloading video JSON')

        title = None
        titles = video.get('titles')
        if titles:
            title = titles.get('en') or titles[titles.keys()[0]]
        if not title:
            title = 'Episode %d' % video.get('number') if video.get('type') == 'episode' else video.get('id') or video_id
            container_titles = video.get('container', {}).get('titles')
            if container_titles:
                container_title = container_titles.get('en') or container_titles[container_titles.keys()[0]]
                title = '%s - %s' % (container_title, title)

        descriptions = video.get('descriptions')
        description = descriptions.get('en') or descriptions[titles.keys()[0]] if descriptions else None

        duration = int_or_none(video.get('duration'))
        timestamp = parse_iso8601(video.get('created_at'))
        uploader = video.get('author')
        like_count = int_or_none(video.get('likes', {}).get('count'))
        age_limit = parse_age_limit(video.get('rating'))

        thumbnails = []
        for thumbnail_id, thumbnail in video.get('images', {}).items():
            thumbnails.append({
                'id': thumbnail_id,
                'url': thumbnail.get('url'),
            })

        subtitles = {}
        for subtitle_lang, _ in video.get('subtitle_completions', {}).items():
            subtitles[subtitle_lang] = [{
                'ext': subtitles_format,
                'url': self._prepare_call(
                    'videos/%s/subtitles/%s.%s' % (video_id, subtitle_lang, subtitles_format)),
            } for subtitles_format in ('srt', 'vtt')]

        result = {
            'id': video_id,
            'title': title,
            'description': description,
            'duration': duration,
            'timestamp': timestamp,
            'uploader': uploader,
            'like_count': like_count,
            'age_limit': age_limit,
            'thumbnails': thumbnails,
            'subtitles': subtitles,
        }

        streams = self._call_api(
            'videos/%s/streams.json' % video_id, video_id,
            'Downloading video streams JSON')

        if 'external' in streams:
            result.update({
                '_type': 'url_transparent',
                'url': streams['external']['url'],
            })
            return result

        formats = []
        for format_id, stream_dict in streams.items():
            height = self._search_regex(
                r'^(\d+)[pP]$', format_id, 'height', default=None)
            for protocol, format_dict in stream_dict.items():
                if format_id == 'm3u8':
                    formats = self._extract_m3u8_formats(
                        format_dict['url'], video_id, 'mp4', m3u8_id='m3u8-%s' % protocol)
                else:
                    formats.append({
                        'url': format_dict['url'],
                        'format_id': '%s-%s' % (format_id, protocol),
                        'height': height,
                    })
        self._sort_formats(formats)

        result['formats'] = formats
        return result


class VikiChannelIE(VikiBaseIE):
    IE_NAME = 'viki:channel'
    _VALID_URL = r'%s(?:tv|news|movies|artists)/(?P<id>[0-9]+c)' % VikiBaseIE._VALID_URL_BASE
    _TESTS = [{
        'url': 'http://www.viki.com/tv/50c-boys-over-flowers',
        'info_dict': {
            'id': '50c',
            'title': 'Boys Over Flowers',
            'description': 'md5:ecd3cff47967fe193cff37c0bec52790',
        },
        'playlist_count': 70,
    }, {
        'url': 'http://www.viki.com/tv/1354c-poor-nastya-complete',
        'info_dict': {
            'id': '1354c',
            'title': 'Poor Nastya [COMPLETE]',
            'description': 'md5:05bf5471385aa8b21c18ad450e350525',
        },
        'playlist_count': 127,
    }, {
        'url': 'http://www.viki.com/news/24569c-showbiz-korea',
        'only_matching': True,
    }, {
        'url': 'http://www.viki.com/movies/22047c-pride-and-prejudice-2005',
        'only_matching': True,
    }, {
        'url': 'http://www.viki.com/artists/2141c-shinee',
        'only_matching': True,
    }]

    _PER_PAGE = 25

    def _real_extract(self, url):
        channel_id = self._match_id(url)

        channel = self._call_api(
            'containers/%s.json' % channel_id, channel_id,
            'Downloading channel JSON')

        titles = channel['titles']
        title = titles.get('en') or titles[titles.keys()[0]]

        descriptions = channel['descriptions']
        description = descriptions.get('en') or descriptions[descriptions.keys()[0]]

        entries = []
        for video_type in ('episodes', 'clips', 'movies'):
            for page_num in itertools.count(1):
                page = self._call_api(
                    'containers/%s/%s.json?per_page=%d&sort=number&direction=asc&with_paging=true&page=%d'
                    % (channel_id, video_type, self._PER_PAGE, page_num), channel_id,
                    'Downloading %s JSON page #%d' % (video_type, page_num))
                for video in page['response']:
                    video_id = video['id']
                    entries.append(self.url_result(
                        'http://www.viki.com/videos/%s' % video_id, 'Viki'))
                if not page['pagination']['next']:
                    break

        return self.playlist_result(entries, channel_id, title, description)
Commit	Line	Data
accf79b1	1	# coding: utf-8
cb9722cb PH	2	from __future__ import unicode_literals
cb9722cb PH	3
accf79b1	4	import json
1a83c731 S	5	import time
	6	import hmac
	7	import hashlib
bc56355e	8	import itertools
382ed50e PH	9
382ed50e PH	10	from ..utils import (
6d88bc37	11	ExtractorError,
1a83c731 S	12	int_or_none,
	13	parse_age_limit,
	14	parse_iso8601,
382ed50e	15	)
accf79b1	16	from ..compat import compat_urllib_request
4f7cea6c	17	from .common import InfoExtractor
382ed50e PH	18
382ed50e PH	19
1a83c731	20	class VikiBaseIE(InfoExtractor):
53de95da	21	_VALID_URL_BASE = r'https?://(?:www\.)?viki\.(?:com\|net\|mx\|jp\|fr)/'
1a83c731 S	22	_API_QUERY_TEMPLATE = '/v4/%sapp=%s&t=%s&site=www.viki.com'
	23	_API_URL_TEMPLATE = 'http://api.viki.io%s&sig=%s'
	24
	25	_APP = '65535a'
	26	_APP_VERSION = '2.2.5.1428709186'
	27	_APP_SECRET = '-$iJ}@p7!G@SyU/je1bEyWg}upLu-6V6-Lg9VD(]siH,r.,m-r\|ulZ,U4LC/SeR)'
	28
accf79b1 S	29	_NETRC_MACHINE = 'viki'
	30
	31	def _prepare_call(self, path, timestamp=None, post_data=None):
1a83c731 S	32	path += '?' if '?' not in path else '&'
	33	if not timestamp:
	34	timestamp = int(time.time())
	35	query = self._API_QUERY_TEMPLATE % (path, self._APP, timestamp)
	36	sig = hmac.new(
	37	self._APP_SECRET.encode('ascii'),
	38	query.encode('ascii'),
	39	hashlib.sha1
	40	).hexdigest()
accf79b1 S	41	url = self._API_URL_TEMPLATE % (query, sig)
	42	return compat_urllib_request.Request(
	43	url, json.dumps(post_data).encode('utf-8')) if post_data else url
1a83c731	44
accf79b1	45	def _call_api(self, path, video_id, note, timestamp=None, post_data=None):
1a83c731	46	resp = self._download_json(
accf79b1	47	self._prepare_call(path, timestamp, post_data), video_id, note)
1a83c731 S	48
	49	error = resp.get('error')
	50	if error:
	51	if error == 'invalid timestamp':
	52	resp = self._download_json(
accf79b1	53	self._prepare_call(path, int(resp['current_timestamp']), post_data),
1a83c731 S	54	video_id, '%s (retry)' % note)
	55	error = resp.get('error')
	56	if error:
	57	self._raise_error(resp['error'])
	58
	59	return resp
	60
	61	def _raise_error(self, error):
	62	raise ExtractorError(
	63	'%s returned error: %s' % (self.IE_NAME, error),
	64	expected=True)
	65
accf79b1 S	66	def _real_initialize(self):
	67	self._login()
	68
	69	def _login(self):
	70	(username, password) = self._get_login_info()
	71	if username is None:
	72	return
	73
	74	login_form = {
	75	'login_id': username,
	76	'password': password,
	77	}
	78
	79	self._call_api(
	80	'sessions.json', None,
	81	'Logging in as %s' % username, post_data=login_form)
	82
1a83c731 S	83
1a83c731 S	84	class VikiIE(VikiBaseIE):
cb9722cb	85	IE_NAME = 'viki'
53de95da	86	_VALID_URL = r'%s(?:videos\|player)/(?P<id>[0-9]+v)' % VikiBaseIE._VALID_URL_BASE
8e3df9df	87	_TESTS = [{
cb9722cb	88	'url': 'http://www.viki.com/videos/1023585v-heirs-episode-14',
cb9722cb PH	89	'info_dict': {
	90	'id': '1023585v',
	91	'ext': 'mp4',
	92	'title': 'Heirs Episode 14',
	93	'uploader': 'SBS',
	94	'description': 'md5:c4b17b9626dd4b143dcc4d855ba3474e',
	95	'upload_date': '20131121',
	96	'age_limit': 13,
6d88bc37	97	},
cb9722cb	98	'skip': 'Blocked in the US',
8e3df9df	99	}, {
1a83c731	100	# clip
8e3df9df	101	'url': 'http://www.viki.com/videos/1067139v-the-avengers-age-of-ultron-press-conference',
1a83c731	102	'md5': '86c0b5dbd4d83a6611a79987cc7a1989',
8e3df9df YCH	103	'info_dict': {
	104	'id': '1067139v',
	105	'ext': 'mp4',
1a83c731	106	'title': "'The Avengers: Age of Ultron' Press Conference",
8e3df9df	107	'description': 'md5:d70b2f9428f5488321bfe1db10d612ea',
1a83c731 S	108	'duration': 352,
1a83c731 S	109	'timestamp': 1430380829,
8e3df9df	110	'upload_date': '20150430',
1a83c731 S	111	'uploader': 'Arirang TV',
	112	'like_count': int,
	113	'age_limit': 0,
8e3df9df	114	}
d948e09b YCH	115	}, {
	116	'url': 'http://www.viki.com/videos/1048879v-ankhon-dekhi',
	117	'info_dict': {
	118	'id': '1048879v',
	119	'ext': 'mp4',
d948e09b	120	'title': 'Ankhon Dekhi',
1a83c731 S	121	'duration': 6512,
	122	'timestamp': 1408532356,
	123	'upload_date': '20140820',
	124	'uploader': 'Spuul',
	125	'like_count': int,
	126	'age_limit': 13,
d948e09b YCH	127	},
d948e09b YCH	128	'params': {
1a83c731	129	# m3u8 download
d948e09b YCH	130	'skip_download': True,
d948e09b YCH	131	}
1a83c731 S	132	}, {
	133	# episode
	134	'url': 'http://www.viki.com/videos/44699v-boys-over-flowers-episode-1',
	135	'md5': '190f3ef426005ba3a080a63325955bc3',
	136	'info_dict': {
	137	'id': '44699v',
	138	'ext': 'mp4',
	139	'title': 'Boys Over Flowers - Episode 1',
	140	'description': 'md5:52617e4f729c7d03bfd4bcbbb6e946f2',
	141	'duration': 4155,
	142	'timestamp': 1270496524,
	143	'upload_date': '20100405',
	144	'uploader': 'group8',
	145	'like_count': int,
	146	'age_limit': 13,
	147	}
ac20d95f S	148	}, {
	149	# youtube external
	150	'url': 'http://www.viki.com/videos/50562v-poor-nastya-complete-episode-1',
	151	'md5': '216d1afdc0c64d1febc1e9f2bd4b864b',
	152	'info_dict': {
	153	'id': '50562v',
	154	'ext': 'mp4',
	155	'title': 'Poor Nastya [COMPLETE] - Episode 1',
	156	'description': '',
	157	'duration': 607,
	158	'timestamp': 1274949505,
	159	'upload_date': '20101213',
	160	'uploader': 'ad14065n',
	161	'uploader_id': 'ad14065n',
	162	'like_count': int,
	163	'age_limit': 13,
	164	}
1a83c731 S	165	}, {
	166	'url': 'http://www.viki.com/player/44699v',
	167	'only_matching': True,
8e3df9df	168	}]
382ed50e PH	169
382ed50e PH	170	def _real_extract(self, url):
8ee34150	171	video_id = self._match_id(url)
382ed50e	172
1a83c731 S	173	video = self._call_api(
	174	'videos/%s.json' % video_id, video_id, 'Downloading video JSON')
	175
	176	title = None
	177	titles = video.get('titles')
	178	if titles:
	179	title = titles.get('en') or titles[titles.keys()[0]]
	180	if not title:
	181	title = 'Episode %d' % video.get('number') if video.get('type') == 'episode' else video.get('id') or video_id
	182	container_titles = video.get('container', {}).get('titles')
	183	if container_titles:
4d8ee013	184	container_title = container_titles.get('en') or container_titles[container_titles.keys()[0]]
1a83c731 S	185	title = '%s - %s' % (container_title, title)
	186
	187	descriptions = video.get('descriptions')
	188	description = descriptions.get('en') or descriptions[titles.keys()[0]] if descriptions else None
	189
	190	duration = int_or_none(video.get('duration'))
	191	timestamp = parse_iso8601(video.get('created_at'))
	192	uploader = video.get('author')
	193	like_count = int_or_none(video.get('likes', {}).get('count'))
	194	age_limit = parse_age_limit(video.get('rating'))
	195
	196	thumbnails = []
	197	for thumbnail_id, thumbnail in video.get('images', {}).items():
	198	thumbnails.append({
	199	'id': thumbnail_id,
	200	'url': thumbnail.get('url'),
	201	})
	202
	203	subtitles = {}
	204	for subtitle_lang, _ in video.get('subtitle_completions', {}).items():
	205	subtitles[subtitle_lang] = [{
	206	'ext': subtitles_format,
	207	'url': self._prepare_call(
	208	'videos/%s/subtitles/%s.%s' % (video_id, subtitle_lang, subtitles_format)),
	209	} for subtitles_format in ('srt', 'vtt')]
382ed50e	210
ac20d95f	211	result = {
382ed50e PH	212	'id': video_id,
382ed50e PH	213	'title': title,
382ed50e	214	'description': description,
1a83c731 S	215	'duration': duration,
1a83c731 S	216	'timestamp': timestamp,
382ed50e	217	'uploader': uploader,
1a83c731 S	218	'like_count': like_count,
	219	'age_limit': age_limit,
	220	'thumbnails': thumbnails,
1a83c731	221	'subtitles': subtitles,
382ed50e PH	222	}
382ed50e PH	223
ac20d95f S	224	streams = self._call_api(
	225	'videos/%s/streams.json' % video_id, video_id,
	226	'Downloading video streams JSON')
	227
	228	if 'external' in streams:
	229	result.update({
	230	'_type': 'url_transparent',
	231	'url': streams['external']['url'],
	232	})
	233	return result
	234
	235	formats = []
	236	for format_id, stream_dict in streams.items():
	237	height = self._search_regex(
	238	r'^(\d+)[pP]$', format_id, 'height', default=None)
	239	for protocol, format_dict in stream_dict.items():
	240	if format_id == 'm3u8':
	241	formats = self._extract_m3u8_formats(
	242	format_dict['url'], video_id, 'mp4', m3u8_id='m3u8-%s' % protocol)
	243	else:
	244	formats.append({
	245	'url': format_dict['url'],
	246	'format_id': '%s-%s' % (format_id, protocol),
	247	'height': height,
	248	})
	249	self._sort_formats(formats)
	250
	251	result['formats'] = formats
	252	return result
	253
0d7f0364	254
bc56355e	255	class VikiChannelIE(VikiBaseIE):
8da0e0e9	256	IE_NAME = 'viki:channel'
53de95da	257	_VALID_URL = r'%s(?:tv\|news\|movies\|artists)/(?P<id>[0-9]+c)' % VikiBaseIE._VALID_URL_BASE
0d7f0364	258	_TESTS = [{
	259	'url': 'http://www.viki.com/tv/50c-boys-over-flowers',
	260	'info_dict': {
	261	'id': '50c',
	262	'title': 'Boys Over Flowers',
	263	'description': 'md5:ecd3cff47967fe193cff37c0bec52790',
	264	},
1c18de00	265	'playlist_count': 70,
	266	}, {
	267	'url': 'http://www.viki.com/tv/1354c-poor-nastya-complete',
	268	'info_dict': {
	269	'id': '1354c',
	270	'title': 'Poor Nastya [COMPLETE]',
	271	'description': 'md5:05bf5471385aa8b21c18ad450e350525',
	272	},
	273	'playlist_count': 127,
d01924f4 S	274	}, {
	275	'url': 'http://www.viki.com/news/24569c-showbiz-korea',
	276	'only_matching': True,
	277	}, {
	278	'url': 'http://www.viki.com/movies/22047c-pride-and-prejudice-2005',
	279	'only_matching': True,
	280	}, {
	281	'url': 'http://www.viki.com/artists/2141c-shinee',
	282	'only_matching': True,
0d7f0364	283	}]
bc56355e	284
8da0e0e9	285	_PER_PAGE = 25
0d7f0364	286
0d7f0364	287	def _real_extract(self, url):
b0d619fd	288	channel_id = self._match_id(url)
0d7f0364	289
bc56355e S	290	channel = self._call_api(
	291	'containers/%s.json' % channel_id, channel_id,
	292	'Downloading channel JSON')
b0d619fd S	293
	294	titles = channel['titles']
	295	title = titles.get('en') or titles[titles.keys()[0]]
	296
	297	descriptions = channel['descriptions']
	298	description = descriptions.get('en') or descriptions[descriptions.keys()[0]]
0d7f0364	299
0d7f0364	300	entries = []
d01924f4	301	for video_type in ('episodes', 'clips', 'movies'):
bc56355e S	302	for page_num in itertools.count(1):
	303	page = self._call_api(
	304	'containers/%s/%s.json?per_page=%d&sort=number&direction=asc&with_paging=true&page=%d'
	305	% (channel_id, video_type, self._PER_PAGE, page_num), channel_id,
	306	'Downloading %s JSON page #%d' % (video_type, page_num))
b0d619fd	307	for video in page['response']:
1c18de00	308	video_id = video['id']
1c18de00	309	entries.append(self.url_result(
bc56355e S	310	'http://www.viki.com/videos/%s' % video_id, 'Viki'))
	311	if not page['pagination']['next']:
	312	break
0d7f0364	313
b0d619fd	314	return self.playlist_result(entries, channel_id, title, description)