[yt-dlp.git] / youtube_dl / extractor / viki.py

# coding: utf-8
from __future__ import unicode_literals

import json
import time
import hmac
import hashlib
import itertools

from ..utils import (
    ExtractorError,
    int_or_none,
    parse_age_limit,
    parse_iso8601,
)
from ..compat import compat_urllib_request
from .common import InfoExtractor


class VikiBaseIE(InfoExtractor):
    _VALID_URL_BASE = r'https?://(?:www\.)?viki\.(?:com|net|mx|jp|fr)/'
    _API_QUERY_TEMPLATE = '/v4/%sapp=%s&t=%s&site=www.viki.com'
    _API_URL_TEMPLATE = 'http://api.viki.io%s&sig=%s'

    _APP = '65535a'
    _APP_VERSION = '2.2.5.1428709186'
    _APP_SECRET = '-$iJ}@p7!G@SyU/je1bEyWg}upLu-6V6-Lg9VD(]siH,r.,m-r|ulZ,U4LC/SeR)'

    _NETRC_MACHINE = 'viki'

    _token = None

    def _prepare_call(self, path, timestamp=None, post_data=None):
        path += '?' if '?' not in path else '&'
        if not timestamp:
            timestamp = int(time.time())
        query = self._API_QUERY_TEMPLATE % (path, self._APP, timestamp)
        if self._token:
            query += '&token=%s' % self._token
        sig = hmac.new(
            self._APP_SECRET.encode('ascii'),
            query.encode('ascii'),
            hashlib.sha1
        ).hexdigest()
        url = self._API_URL_TEMPLATE % (query, sig)
        return compat_urllib_request.Request(
            url, json.dumps(post_data).encode('utf-8')) if post_data else url

    def _call_api(self, path, video_id, note, timestamp=None, post_data=None):
        resp = self._download_json(
            self._prepare_call(path, timestamp, post_data), video_id, note)

        error = resp.get('error')
        if error:
            if error == 'invalid timestamp':
                resp = self._download_json(
                    self._prepare_call(path, int(resp['current_timestamp']), post_data),
                    video_id, '%s (retry)' % note)
                error = resp.get('error')
            if error:
                self._raise_error(resp['error'])

        return resp

    def _raise_error(self, error):
        raise ExtractorError(
            '%s returned error: %s' % (self.IE_NAME, error),
            expected=True)

    def _real_initialize(self):
        self._login()

    def _login(self):
        (username, password) = self._get_login_info()
        if username is None:
            return

        login_form = {
            'login_id': username,
            'password': password,
        }

        login = self._call_api(
            'sessions.json', None,
            'Logging in as %s' % username, post_data=login_form)

        self._token = login.get('token')
        if not self._token:
            self.report_warning('Unable to get session token, login has probably failed')


class VikiIE(VikiBaseIE):
    IE_NAME = 'viki'
    _VALID_URL = r'%s(?:videos|player)/(?P<id>[0-9]+v)' % VikiBaseIE._VALID_URL_BASE
    _TESTS = [{
        'url': 'http://www.viki.com/videos/1023585v-heirs-episode-14',
        'info_dict': {
            'id': '1023585v',
            'ext': 'mp4',
            'title': 'Heirs Episode 14',
            'uploader': 'SBS',
            'description': 'md5:c4b17b9626dd4b143dcc4d855ba3474e',
            'upload_date': '20131121',
            'age_limit': 13,
        },
        'skip': 'Blocked in the US',
    }, {
        # clip
        'url': 'http://www.viki.com/videos/1067139v-the-avengers-age-of-ultron-press-conference',
        'md5': '86c0b5dbd4d83a6611a79987cc7a1989',
        'info_dict': {
            'id': '1067139v',
            'ext': 'mp4',
            'title': "'The Avengers: Age of Ultron' Press Conference",
            'description': 'md5:d70b2f9428f5488321bfe1db10d612ea',
            'duration': 352,
            'timestamp': 1430380829,
            'upload_date': '20150430',
            'uploader': 'Arirang TV',
            'like_count': int,
            'age_limit': 0,
        }
    }, {
        'url': 'http://www.viki.com/videos/1048879v-ankhon-dekhi',
        'info_dict': {
            'id': '1048879v',
            'ext': 'mp4',
            'title': 'Ankhon Dekhi',
            'duration': 6512,
            'timestamp': 1408532356,
            'upload_date': '20140820',
            'uploader': 'Spuul',
            'like_count': int,
            'age_limit': 13,
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        }
    }, {
        # episode
        'url': 'http://www.viki.com/videos/44699v-boys-over-flowers-episode-1',
        'md5': '190f3ef426005ba3a080a63325955bc3',
        'info_dict': {
            'id': '44699v',
            'ext': 'mp4',
            'title': 'Boys Over Flowers - Episode 1',
            'description': 'md5:52617e4f729c7d03bfd4bcbbb6e946f2',
            'duration': 4155,
            'timestamp': 1270496524,
            'upload_date': '20100405',
            'uploader': 'group8',
            'like_count': int,
            'age_limit': 13,
        }
    }, {
        # youtube external
        'url': 'http://www.viki.com/videos/50562v-poor-nastya-complete-episode-1',
        'md5': '216d1afdc0c64d1febc1e9f2bd4b864b',
        'info_dict': {
            'id': '50562v',
            'ext': 'mp4',
            'title': 'Poor Nastya [COMPLETE] - Episode 1',
            'description': '',
            'duration': 607,
            'timestamp': 1274949505,
            'upload_date': '20101213',
            'uploader': 'ad14065n',
            'uploader_id': 'ad14065n',
            'like_count': int,
            'age_limit': 13,
        }
    }, {
        'url': 'http://www.viki.com/player/44699v',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        video = self._call_api(
            'videos/%s.json' % video_id, video_id, 'Downloading video JSON')

        title = None
        titles = video.get('titles')
        if titles:
            title = titles.get('en') or titles[titles.keys()[0]]
        if not title:
            title = 'Episode %d' % video.get('number') if video.get('type') == 'episode' else video.get('id') or video_id
            container_titles = video.get('container', {}).get('titles')
            if container_titles:
                container_title = container_titles.get('en') or container_titles[container_titles.keys()[0]]
                title = '%s - %s' % (container_title, title)

        descriptions = video.get('descriptions')
        description = descriptions.get('en') or descriptions[titles.keys()[0]] if descriptions else None

        duration = int_or_none(video.get('duration'))
        timestamp = parse_iso8601(video.get('created_at'))
        uploader = video.get('author')
        like_count = int_or_none(video.get('likes', {}).get('count'))
        age_limit = parse_age_limit(video.get('rating'))

        thumbnails = []
        for thumbnail_id, thumbnail in video.get('images', {}).items():
            thumbnails.append({
                'id': thumbnail_id,
                'url': thumbnail.get('url'),
            })

        subtitles = {}
        for subtitle_lang, _ in video.get('subtitle_completions', {}).items():
            subtitles[subtitle_lang] = [{
                'ext': subtitles_format,
                'url': self._prepare_call(
                    'videos/%s/subtitles/%s.%s' % (video_id, subtitle_lang, subtitles_format)),
            } for subtitles_format in ('srt', 'vtt')]

        result = {
            'id': video_id,
            'title': title,
            'description': description,
            'duration': duration,
            'timestamp': timestamp,
            'uploader': uploader,
            'like_count': like_count,
            'age_limit': age_limit,
            'thumbnails': thumbnails,
            'subtitles': subtitles,
        }

        streams = self._call_api(
            'videos/%s/streams.json' % video_id, video_id,
            'Downloading video streams JSON')

        if 'external' in streams:
            result.update({
                '_type': 'url_transparent',
                'url': streams['external']['url'],
            })
            return result

        formats = []
        for format_id, stream_dict in streams.items():
            height = int_or_none(self._search_regex(
                r'^(\d+)[pP]$', format_id, 'height', default=None))
            for protocol, format_dict in stream_dict.items():
                if format_id == 'm3u8':
                    formats = self._extract_m3u8_formats(
                        format_dict['url'], video_id, 'mp4', m3u8_id='m3u8-%s' % protocol)
                else:
                    formats.append({
                        'url': format_dict['url'],
                        'format_id': '%s-%s' % (format_id, protocol),
                        'height': height,
                    })
        self._sort_formats(formats)

        result['formats'] = formats
        return result


class VikiChannelIE(VikiBaseIE):
    IE_NAME = 'viki:channel'
    _VALID_URL = r'%s(?:tv|news|movies|artists)/(?P<id>[0-9]+c)' % VikiBaseIE._VALID_URL_BASE
    _TESTS = [{
        'url': 'http://www.viki.com/tv/50c-boys-over-flowers',
        'info_dict': {
            'id': '50c',
            'title': 'Boys Over Flowers',
            'description': 'md5:ecd3cff47967fe193cff37c0bec52790',
        },
        'playlist_count': 70,
    }, {
        'url': 'http://www.viki.com/tv/1354c-poor-nastya-complete',
        'info_dict': {
            'id': '1354c',
            'title': 'Poor Nastya [COMPLETE]',
            'description': 'md5:05bf5471385aa8b21c18ad450e350525',
        },
        'playlist_count': 127,
    }, {
        'url': 'http://www.viki.com/news/24569c-showbiz-korea',
        'only_matching': True,
    }, {
        'url': 'http://www.viki.com/movies/22047c-pride-and-prejudice-2005',
        'only_matching': True,
    }, {
        'url': 'http://www.viki.com/artists/2141c-shinee',
        'only_matching': True,
    }]

    _PER_PAGE = 25

    def _real_extract(self, url):
        channel_id = self._match_id(url)

        channel = self._call_api(
            'containers/%s.json' % channel_id, channel_id,
            'Downloading channel JSON')

        titles = channel['titles']
        title = titles.get('en') or titles[titles.keys()[0]]

        descriptions = channel['descriptions']
        description = descriptions.get('en') or descriptions[descriptions.keys()[0]]

        entries = []
        for video_type in ('episodes', 'clips', 'movies'):
            for page_num in itertools.count(1):
                page = self._call_api(
                    'containers/%s/%s.json?per_page=%d&sort=number&direction=asc&with_paging=true&page=%d'
                    % (channel_id, video_type, self._PER_PAGE, page_num), channel_id,
                    'Downloading %s JSON page #%d' % (video_type, page_num))
                for video in page['response']:
                    video_id = video['id']
                    entries.append(self.url_result(
                        'http://www.viki.com/videos/%s' % video_id, 'Viki'))
                if not page['pagination']['next']:
                    break

        return self.playlist_result(entries, channel_id, title, description)
Commit	Line	Data
accf79b1	1	# coding: utf-8
cb9722cb PH	2	from __future__ import unicode_literals
cb9722cb PH	3
accf79b1	4	import json
1a83c731 S	5	import time
	6	import hmac
	7	import hashlib
bc56355e	8	import itertools
382ed50e PH	9
382ed50e PH	10	from ..utils import (
6d88bc37	11	ExtractorError,
1a83c731 S	12	int_or_none,
	13	parse_age_limit,
	14	parse_iso8601,
382ed50e	15	)
accf79b1	16	from ..compat import compat_urllib_request
4f7cea6c	17	from .common import InfoExtractor
382ed50e PH	18
382ed50e PH	19
1a83c731	20	class VikiBaseIE(InfoExtractor):
53de95da	21	_VALID_URL_BASE = r'https?://(?:www\.)?viki\.(?:com\|net\|mx\|jp\|fr)/'
1a83c731 S	22	_API_QUERY_TEMPLATE = '/v4/%sapp=%s&t=%s&site=www.viki.com'
	23	_API_URL_TEMPLATE = 'http://api.viki.io%s&sig=%s'
	24
	25	_APP = '65535a'
	26	_APP_VERSION = '2.2.5.1428709186'
	27	_APP_SECRET = '-$iJ}@p7!G@SyU/je1bEyWg}upLu-6V6-Lg9VD(]siH,r.,m-r\|ulZ,U4LC/SeR)'
	28
accf79b1 S	29	_NETRC_MACHINE = 'viki'
accf79b1 S	30
16d6973f S	31	_token = None
16d6973f S	32
accf79b1	33	def _prepare_call(self, path, timestamp=None, post_data=None):
1a83c731 S	34	path += '?' if '?' not in path else '&'
	35	if not timestamp:
	36	timestamp = int(time.time())
	37	query = self._API_QUERY_TEMPLATE % (path, self._APP, timestamp)
16d6973f S	38	if self._token:
16d6973f S	39	query += '&token=%s' % self._token
1a83c731 S	40	sig = hmac.new(
	41	self._APP_SECRET.encode('ascii'),
	42	query.encode('ascii'),
	43	hashlib.sha1
	44	).hexdigest()
accf79b1 S	45	url = self._API_URL_TEMPLATE % (query, sig)
	46	return compat_urllib_request.Request(
	47	url, json.dumps(post_data).encode('utf-8')) if post_data else url
1a83c731	48
accf79b1	49	def _call_api(self, path, video_id, note, timestamp=None, post_data=None):
1a83c731	50	resp = self._download_json(
accf79b1	51	self._prepare_call(path, timestamp, post_data), video_id, note)
1a83c731 S	52
	53	error = resp.get('error')
	54	if error:
	55	if error == 'invalid timestamp':
	56	resp = self._download_json(
accf79b1	57	self._prepare_call(path, int(resp['current_timestamp']), post_data),
1a83c731 S	58	video_id, '%s (retry)' % note)
	59	error = resp.get('error')
	60	if error:
	61	self._raise_error(resp['error'])
	62
	63	return resp
	64
	65	def _raise_error(self, error):
	66	raise ExtractorError(
	67	'%s returned error: %s' % (self.IE_NAME, error),
	68	expected=True)
	69
accf79b1 S	70	def _real_initialize(self):
	71	self._login()
	72
	73	def _login(self):
	74	(username, password) = self._get_login_info()
	75	if username is None:
	76	return
	77
	78	login_form = {
	79	'login_id': username,
	80	'password': password,
	81	}
	82
16d6973f	83	login = self._call_api(
accf79b1 S	84	'sessions.json', None,
	85	'Logging in as %s' % username, post_data=login_form)
	86
16d6973f S	87	self._token = login.get('token')
	88	if not self._token:
	89	self.report_warning('Unable to get session token, login has probably failed')
	90
1a83c731 S	91
1a83c731 S	92	class VikiIE(VikiBaseIE):
cb9722cb	93	IE_NAME = 'viki'
53de95da	94	_VALID_URL = r'%s(?:videos\|player)/(?P<id>[0-9]+v)' % VikiBaseIE._VALID_URL_BASE
8e3df9df	95	_TESTS = [{
cb9722cb	96	'url': 'http://www.viki.com/videos/1023585v-heirs-episode-14',
cb9722cb PH	97	'info_dict': {
	98	'id': '1023585v',
	99	'ext': 'mp4',
	100	'title': 'Heirs Episode 14',
	101	'uploader': 'SBS',
	102	'description': 'md5:c4b17b9626dd4b143dcc4d855ba3474e',
	103	'upload_date': '20131121',
	104	'age_limit': 13,
6d88bc37	105	},
cb9722cb	106	'skip': 'Blocked in the US',
8e3df9df	107	}, {
1a83c731	108	# clip
8e3df9df	109	'url': 'http://www.viki.com/videos/1067139v-the-avengers-age-of-ultron-press-conference',
1a83c731	110	'md5': '86c0b5dbd4d83a6611a79987cc7a1989',
8e3df9df YCH	111	'info_dict': {
	112	'id': '1067139v',
	113	'ext': 'mp4',
1a83c731	114	'title': "'The Avengers: Age of Ultron' Press Conference",
8e3df9df	115	'description': 'md5:d70b2f9428f5488321bfe1db10d612ea',
1a83c731 S	116	'duration': 352,
1a83c731 S	117	'timestamp': 1430380829,
8e3df9df	118	'upload_date': '20150430',
1a83c731 S	119	'uploader': 'Arirang TV',
	120	'like_count': int,
	121	'age_limit': 0,
8e3df9df	122	}
d948e09b YCH	123	}, {
	124	'url': 'http://www.viki.com/videos/1048879v-ankhon-dekhi',
	125	'info_dict': {
	126	'id': '1048879v',
	127	'ext': 'mp4',
d948e09b	128	'title': 'Ankhon Dekhi',
1a83c731 S	129	'duration': 6512,
	130	'timestamp': 1408532356,
	131	'upload_date': '20140820',
	132	'uploader': 'Spuul',
	133	'like_count': int,
	134	'age_limit': 13,
d948e09b YCH	135	},
d948e09b YCH	136	'params': {
1a83c731	137	# m3u8 download
d948e09b YCH	138	'skip_download': True,
d948e09b YCH	139	}
1a83c731 S	140	}, {
	141	# episode
	142	'url': 'http://www.viki.com/videos/44699v-boys-over-flowers-episode-1',
	143	'md5': '190f3ef426005ba3a080a63325955bc3',
	144	'info_dict': {
	145	'id': '44699v',
	146	'ext': 'mp4',
	147	'title': 'Boys Over Flowers - Episode 1',
	148	'description': 'md5:52617e4f729c7d03bfd4bcbbb6e946f2',
	149	'duration': 4155,
	150	'timestamp': 1270496524,
	151	'upload_date': '20100405',
	152	'uploader': 'group8',
	153	'like_count': int,
	154	'age_limit': 13,
	155	}
ac20d95f S	156	}, {
	157	# youtube external
	158	'url': 'http://www.viki.com/videos/50562v-poor-nastya-complete-episode-1',
	159	'md5': '216d1afdc0c64d1febc1e9f2bd4b864b',
	160	'info_dict': {
	161	'id': '50562v',
	162	'ext': 'mp4',
	163	'title': 'Poor Nastya [COMPLETE] - Episode 1',
	164	'description': '',
	165	'duration': 607,
	166	'timestamp': 1274949505,
	167	'upload_date': '20101213',
	168	'uploader': 'ad14065n',
	169	'uploader_id': 'ad14065n',
	170	'like_count': int,
	171	'age_limit': 13,
	172	}
1a83c731 S	173	}, {
	174	'url': 'http://www.viki.com/player/44699v',
	175	'only_matching': True,
8e3df9df	176	}]
382ed50e PH	177
382ed50e PH	178	def _real_extract(self, url):
8ee34150	179	video_id = self._match_id(url)
382ed50e	180
1a83c731 S	181	video = self._call_api(
	182	'videos/%s.json' % video_id, video_id, 'Downloading video JSON')
	183
	184	title = None
	185	titles = video.get('titles')
	186	if titles:
	187	title = titles.get('en') or titles[titles.keys()[0]]
	188	if not title:
	189	title = 'Episode %d' % video.get('number') if video.get('type') == 'episode' else video.get('id') or video_id
	190	container_titles = video.get('container', {}).get('titles')
	191	if container_titles:
4d8ee013	192	container_title = container_titles.get('en') or container_titles[container_titles.keys()[0]]
1a83c731 S	193	title = '%s - %s' % (container_title, title)
	194
	195	descriptions = video.get('descriptions')
	196	description = descriptions.get('en') or descriptions[titles.keys()[0]] if descriptions else None
	197
	198	duration = int_or_none(video.get('duration'))
	199	timestamp = parse_iso8601(video.get('created_at'))
	200	uploader = video.get('author')
	201	like_count = int_or_none(video.get('likes', {}).get('count'))
	202	age_limit = parse_age_limit(video.get('rating'))
	203
	204	thumbnails = []
	205	for thumbnail_id, thumbnail in video.get('images', {}).items():
	206	thumbnails.append({
	207	'id': thumbnail_id,
	208	'url': thumbnail.get('url'),
	209	})
	210
	211	subtitles = {}
	212	for subtitle_lang, _ in video.get('subtitle_completions', {}).items():
	213	subtitles[subtitle_lang] = [{
	214	'ext': subtitles_format,
	215	'url': self._prepare_call(
	216	'videos/%s/subtitles/%s.%s' % (video_id, subtitle_lang, subtitles_format)),
	217	} for subtitles_format in ('srt', 'vtt')]
382ed50e	218
ac20d95f	219	result = {
382ed50e PH	220	'id': video_id,
382ed50e PH	221	'title': title,
382ed50e	222	'description': description,
1a83c731 S	223	'duration': duration,
1a83c731 S	224	'timestamp': timestamp,
382ed50e	225	'uploader': uploader,
1a83c731 S	226	'like_count': like_count,
	227	'age_limit': age_limit,
	228	'thumbnails': thumbnails,
1a83c731	229	'subtitles': subtitles,
382ed50e PH	230	}
382ed50e PH	231
ac20d95f S	232	streams = self._call_api(
	233	'videos/%s/streams.json' % video_id, video_id,
	234	'Downloading video streams JSON')
	235
	236	if 'external' in streams:
	237	result.update({
	238	'_type': 'url_transparent',
	239	'url': streams['external']['url'],
	240	})
	241	return result
	242
	243	formats = []
	244	for format_id, stream_dict in streams.items():
c59b61c0 S	245	height = int_or_none(self._search_regex(
c59b61c0 S	246	r'^(\d+)[pP]$', format_id, 'height', default=None))
ac20d95f S	247	for protocol, format_dict in stream_dict.items():
	248	if format_id == 'm3u8':
	249	formats = self._extract_m3u8_formats(
	250	format_dict['url'], video_id, 'mp4', m3u8_id='m3u8-%s' % protocol)
	251	else:
	252	formats.append({
	253	'url': format_dict['url'],
	254	'format_id': '%s-%s' % (format_id, protocol),
	255	'height': height,
	256	})
	257	self._sort_formats(formats)
	258
	259	result['formats'] = formats
	260	return result
	261
0d7f0364	262
bc56355e	263	class VikiChannelIE(VikiBaseIE):
8da0e0e9	264	IE_NAME = 'viki:channel'
53de95da	265	_VALID_URL = r'%s(?:tv\|news\|movies\|artists)/(?P<id>[0-9]+c)' % VikiBaseIE._VALID_URL_BASE
0d7f0364	266	_TESTS = [{
	267	'url': 'http://www.viki.com/tv/50c-boys-over-flowers',
	268	'info_dict': {
	269	'id': '50c',
	270	'title': 'Boys Over Flowers',
	271	'description': 'md5:ecd3cff47967fe193cff37c0bec52790',
	272	},
1c18de00	273	'playlist_count': 70,
	274	}, {
	275	'url': 'http://www.viki.com/tv/1354c-poor-nastya-complete',
	276	'info_dict': {
	277	'id': '1354c',
	278	'title': 'Poor Nastya [COMPLETE]',
	279	'description': 'md5:05bf5471385aa8b21c18ad450e350525',
	280	},
	281	'playlist_count': 127,
d01924f4 S	282	}, {
	283	'url': 'http://www.viki.com/news/24569c-showbiz-korea',
	284	'only_matching': True,
	285	}, {
	286	'url': 'http://www.viki.com/movies/22047c-pride-and-prejudice-2005',
	287	'only_matching': True,
	288	}, {
	289	'url': 'http://www.viki.com/artists/2141c-shinee',
	290	'only_matching': True,
0d7f0364	291	}]
bc56355e	292
8da0e0e9	293	_PER_PAGE = 25
0d7f0364	294
0d7f0364	295	def _real_extract(self, url):
b0d619fd	296	channel_id = self._match_id(url)
0d7f0364	297
bc56355e S	298	channel = self._call_api(
	299	'containers/%s.json' % channel_id, channel_id,
	300	'Downloading channel JSON')
b0d619fd S	301
	302	titles = channel['titles']
	303	title = titles.get('en') or titles[titles.keys()[0]]
	304
	305	descriptions = channel['descriptions']
	306	description = descriptions.get('en') or descriptions[descriptions.keys()[0]]
0d7f0364	307
0d7f0364	308	entries = []
d01924f4	309	for video_type in ('episodes', 'clips', 'movies'):
bc56355e S	310	for page_num in itertools.count(1):
	311	page = self._call_api(
	312	'containers/%s/%s.json?per_page=%d&sort=number&direction=asc&with_paging=true&page=%d'
	313	% (channel_id, video_type, self._PER_PAGE, page_num), channel_id,
	314	'Downloading %s JSON page #%d' % (video_type, page_num))
b0d619fd	315	for video in page['response']:
1c18de00	316	video_id = video['id']
1c18de00	317	entries.append(self.url_result(
bc56355e S	318	'http://www.viki.com/videos/%s' % video_id, 'Viki'))
	319	if not page['pagination']['next']:
	320	break
0d7f0364	321
b0d619fd	322	return self.playlist_result(entries, channel_id, title, description)