youtube_dlc/extractor/viki.py

   1 # coding: utf-8
   2 from __future__ import unicode_literals
   3
   4 import hashlib
   5 import hmac
   6 import itertools
   7 import json
   8 import re
   9 import time
  10
  11 from .common import InfoExtractor
  12 from ..utils import (
  13     ExtractorError,
  14     int_or_none,
  15     HEADRequest,
  16     parse_age_limit,
  17     parse_iso8601,
  18     sanitized_Request,
  19 )
  20
  21
  22 class VikiBaseIE(InfoExtractor):
  23     _VALID_URL_BASE = r'https?://(?:www\.)?viki\.(?:com|net|mx|jp|fr)/'
  24     _API_QUERY_TEMPLATE = '/v4/%sapp=%s&t=%s&site=www.viki.com'
  25     _API_URL_TEMPLATE = 'https://api.viki.io%s&sig=%s'
  26
  27     _APP = '100005a'
  28     _APP_VERSION = '2.2.5.1428709186'
  29     _APP_SECRET = 'MM_d*yP@`&1@]@!AVrXf_o-HVEnoTnm$O-ti4[G~$JDI/Dc-&piU&z&5.;:}95=Iad'
  30
  31     _GEO_BYPASS = False
  32     _NETRC_MACHINE = 'viki'
  33
  34     _token = None
  35
  36     _ERRORS = {
  37         'geo': 'Sorry, this content is not available in your region.',
  38         'upcoming': 'Sorry, this content is not yet available.',
  39         # 'paywall': 'paywall',
  40     }
  41
  42     def _prepare_call(self, path, timestamp=None, post_data=None):
  43         path += '?' if '?' not in path else '&'
  44         if not timestamp:
  45             timestamp = int(time.time())
  46         query = self._API_QUERY_TEMPLATE % (path, self._APP, timestamp)
  47         if self._token:
  48             query += '&token=%s' % self._token
  49         sig = hmac.new(
  50             self._APP_SECRET.encode('ascii'),
  51             query.encode('ascii'),
  52             hashlib.sha1
  53         ).hexdigest()
  54         url = self._API_URL_TEMPLATE % (query, sig)
  55         return sanitized_Request(
  56             url, json.dumps(post_data).encode('utf-8')) if post_data else url
  57
  58     def _call_api(self, path, video_id, note, timestamp=None, post_data=None):
  59         resp = self._download_json(
  60             self._prepare_call(path, timestamp, post_data), video_id, note, headers={'x-viki-app-ver': '2.2.5.1428709186'}, expected_status=[200, 400, 404])
  61
  62         error = resp.get('error')
  63         if error:
  64             if error == 'invalid timestamp':
  65                 resp = self._download_json(
  66                     self._prepare_call(path, int(resp['current_timestamp']), post_data),
  67                     video_id, '%s (retry)' % note, headers={'x-viki-app-ver': '2.2.5.1428709186'}, expected_status=[200, 400, 404])
  68                 error = resp.get('error')
  69             if error:
  70                 self._raise_error(resp['error'])
  71
  72         return resp
  73
  74     def _raise_error(self, error):
  75         raise ExtractorError(
  76             '%s returned error: %s' % (self.IE_NAME, error),
  77             expected=True)
  78
  79     def _check_errors(self, data):
  80         for reason, status in data.get('blocking', {}).items():
  81             if status and reason in self._ERRORS:
  82                 message = self._ERRORS[reason]
  83                 if reason == 'geo':
  84                     self.raise_geo_restricted(msg=message)
  85                 raise ExtractorError('%s said: %s' % (
  86                     self.IE_NAME, message), expected=True)
  87
  88     def _real_initialize(self):
  89         self._login()
  90
  91     def _login(self):
  92         username, password = self._get_login_info()
  93         if username is None:
  94             return
  95
  96         login_form = {
  97             'login_id': username,
  98             'password': password,
  99         }
 100
 101         login = self._call_api(
 102             'sessions.json', None,
 103             'Logging in', post_data=login_form)
 104
 105         self._token = login.get('token')
 106         if not self._token:
 107             self.report_warning('Unable to get session token, login has probably failed')
 108
 109     @staticmethod
 110     def dict_selection(dict_obj, preferred_key, allow_fallback=True):
 111         if preferred_key in dict_obj:
 112             return dict_obj.get(preferred_key)
 113
 114         if not allow_fallback:
 115             return
 116
 117         filtered_dict = list(filter(None, [dict_obj.get(k) for k in dict_obj.keys()]))
 118         return filtered_dict[0] if filtered_dict else None
 119
 120
 121 class VikiIE(VikiBaseIE):
 122     IE_NAME = 'viki'
 123     _VALID_URL = r'%s(?:videos|player)/(?P<id>[0-9]+v)' % VikiBaseIE._VALID_URL_BASE
 124     _TESTS = [{
 125         'url': 'http://www.viki.com/videos/1023585v-heirs-episode-14',
 126         'info_dict': {
 127             'id': '1023585v',
 128             'ext': 'mp4',
 129             'title': 'Heirs Episode 14',
 130             'uploader': 'SBS',
 131             'description': 'md5:c4b17b9626dd4b143dcc4d855ba3474e',
 132             'upload_date': '20131121',
 133             'age_limit': 13,
 134         },
 135         'skip': 'Blocked in the US',
 136     }, {
 137         # clip
 138         'url': 'http://www.viki.com/videos/1067139v-the-avengers-age-of-ultron-press-conference',
 139         'md5': '86c0b5dbd4d83a6611a79987cc7a1989',
 140         'info_dict': {
 141             'id': '1067139v',
 142             'ext': 'mp4',
 143             'title': "'The Avengers: Age of Ultron' Press Conference",
 144             'description': 'md5:d70b2f9428f5488321bfe1db10d612ea',
 145             'duration': 352,
 146             'timestamp': 1430380829,
 147             'upload_date': '20150430',
 148             'uploader': 'Arirang TV',
 149             'like_count': int,
 150             'age_limit': 0,
 151         }
 152     }, {
 153         'url': 'http://www.viki.com/videos/1048879v-ankhon-dekhi',
 154         'info_dict': {
 155             'id': '1048879v',
 156             'ext': 'mp4',
 157             'title': 'Ankhon Dekhi',
 158             'duration': 6512,
 159             'timestamp': 1408532356,
 160             'upload_date': '20140820',
 161             'uploader': 'Spuul',
 162             'like_count': int,
 163             'age_limit': 13,
 164         },
 165         'skip': 'Blocked in the US',
 166     }, {
 167         # episode
 168         'url': 'http://www.viki.com/videos/44699v-boys-over-flowers-episode-1',
 169         'md5': '5fa476a902e902783ac7a4d615cdbc7a',
 170         'info_dict': {
 171             'id': '44699v',
 172             'ext': 'mp4',
 173             'title': 'Boys Over Flowers - Episode 1',
 174             'description': 'md5:b89cf50038b480b88b5b3c93589a9076',
 175             'duration': 4204,
 176             'timestamp': 1270496524,
 177             'upload_date': '20100405',
 178             'uploader': 'group8',
 179             'like_count': int,
 180             'age_limit': 13,
 181         }
 182     }, {
 183         # youtube external
 184         'url': 'http://www.viki.com/videos/50562v-poor-nastya-complete-episode-1',
 185         'md5': '63f8600c1da6f01b7640eee7eca4f1da',
 186         'info_dict': {
 187             'id': '50562v',
 188             'ext': 'webm',
 189             'title': 'Poor Nastya [COMPLETE] - Episode 1',
 190             'description': '',
 191             'duration': 606,
 192             'timestamp': 1274949505,
 193             'upload_date': '20101213',
 194             'uploader': 'ad14065n',
 195             'uploader_id': 'ad14065n',
 196             'like_count': int,
 197             'age_limit': 13,
 198         }
 199     }, {
 200         'url': 'http://www.viki.com/player/44699v',
 201         'only_matching': True,
 202     }, {
 203         # non-English description
 204         'url': 'http://www.viki.com/videos/158036v-love-in-magic',
 205         'md5': '1713ae35df5a521b31f6dc40730e7c9c',
 206         'info_dict': {
 207             'id': '158036v',
 208             'ext': 'mp4',
 209             'uploader': 'I Planet Entertainment',
 210             'upload_date': '20111122',
 211             'timestamp': 1321985454,
 212             'description': 'md5:44b1e46619df3a072294645c770cef36',
 213             'title': 'Love In Magic',
 214             'age_limit': 13,
 215         },
 216     }]
 217
 218     def _real_extract(self, url):
 219         video_id = self._match_id(url)
 220
 221         video = self._call_api(
 222             'videos/%s.json' % video_id, video_id, 'Downloading video JSON')
 223
 224         streams = self._call_api(
 225             'videos/%s/streams.json' % video_id, video_id,
 226             'Downloading video streams JSON')
 227
 228         formats = []
 229         for format_id, stream_dict in streams.items():
 230             height = int_or_none(self._search_regex(
 231                 r'^(\d+)[pP]$', format_id, 'height', default=None))
 232             for protocol, format_dict in stream_dict.items():
 233                 # rtmps URLs does not seem to work
 234                 if protocol == 'rtmps':
 235                     continue
 236                 format_url = format_dict.get('url')
 237                 format_drms = format_dict.get('drms')
 238                 format_stream_id = format_dict.get('id')
 239                 if format_id == 'm3u8':
 240                     m3u8_formats = self._extract_m3u8_formats(
 241                         format_url, video_id, 'mp4',
 242                         entry_protocol='m3u8_native',
 243                         m3u8_id='m3u8-%s' % protocol, fatal=False)
 244                     # Despite CODECS metadata in m3u8 all video-only formats
 245                     # are actually video+audio
 246                     for f in m3u8_formats:
 247                         if f.get('acodec') == 'none' and f.get('vcodec') != 'none':
 248                             f['acodec'] = None
 249                     formats.extend(m3u8_formats)
 250                 elif format_id == 'mpd':
 251                     mpd_formats = self._extract_mpd_formats(
 252                         format_url, video_id,
 253                         mpd_id='mpd-%s' % protocol, fatal=False)
 254                     formats.extend(mpd_formats)
 255                 elif format_id == 'mpd':
 256
 257                     formats.extend(mpd_formats)
 258                 elif format_url.startswith('rtmp'):
 259                     mobj = re.search(
 260                         r'^(?P<url>rtmp://[^/]+/(?P<app>.+?))/(?P<playpath>mp4:.+)$',
 261                         format_url)
 262                     if not mobj:
 263                         continue
 264                     formats.append({
 265                         'format_id': 'rtmp-%s' % format_id,
 266                         'ext': 'flv',
 267                         'url': mobj.group('url'),
 268                         'play_path': mobj.group('playpath'),
 269                         'app': mobj.group('app'),
 270                         'page_url': url,
 271                         'drms': format_drms,
 272                         'stream_id': format_stream_id,
 273                     })
 274                 else:
 275                     urlh = self._request_webpage(
 276                         HEADRequest(format_url), video_id, 'Checking file size', fatal=False)
 277                     formats.append({
 278                         'url': format_url,
 279                         'format_id': '%s-%s' % (format_id, protocol),
 280                         'height': height,
 281                         'drms': format_drms,
 282                         'stream_id': format_stream_id,
 283                         'filesize': int_or_none(urlh.headers.get('Content-Length')),
 284                     })
 285         self._sort_formats(formats)
 286
 287         self._check_errors(video)
 288
 289         title = self.dict_selection(video.get('titles', {}), 'en', allow_fallback=False)
 290         if not title:
 291             title = 'Episode %d' % video.get('number') if video.get('type') == 'episode' else video.get('id') or video_id
 292             container_titles = video.get('container', {}).get('titles', {})
 293             container_title = self.dict_selection(container_titles, 'en')
 294             title = '%s - %s' % (container_title, title)
 295
 296         description = self.dict_selection(video.get('descriptions', {}), 'en')
 297
 298         duration = int_or_none(video.get('duration'))
 299         timestamp = parse_iso8601(video.get('created_at'))
 300         uploader = video.get('author')
 301         like_count = int_or_none(video.get('likes', {}).get('count'))
 302         age_limit = parse_age_limit(video.get('rating'))
 303
 304         thumbnails = []
 305         for thumbnail_id, thumbnail in video.get('images', {}).items():
 306             thumbnails.append({
 307                 'id': thumbnail_id,
 308                 'url': thumbnail.get('url'),
 309             })
 310
 311         stream_ids = []
 312         for f in formats:
 313             s_id = f.get('stream_id')
 314             if s_id is not None:
 315                 stream_ids.append(s_id)
 316
 317         subtitles = {}
 318         for subtitle_lang, _ in video.get('subtitle_completions', {}).items():
 319             subtitles[subtitle_lang] = [{
 320                 'ext': subtitles_format,
 321                 'url': self._prepare_call(
 322                     'videos/%s/subtitles/%s.%s?stream_id=%s' % (video_id, subtitle_lang, subtitles_format, stream_ids[0])),
 323             } for subtitles_format in ('srt', 'vtt')]
 324
 325         result = {
 326             'id': video_id,
 327             'title': title,
 328             'description': description,
 329             'duration': duration,
 330             'timestamp': timestamp,
 331             'uploader': uploader,
 332             'like_count': like_count,
 333             'age_limit': age_limit,
 334             'thumbnails': thumbnails,
 335             'subtitles': subtitles,
 336         }
 337
 338         if 'external' in streams:
 339             result.update({
 340                 '_type': 'url_transparent',
 341                 'url': streams['external']['url'],
 342             })
 343             return result
 344
 345         result['formats'] = formats
 346         return result
 347
 348
 349 class VikiChannelIE(VikiBaseIE):
 350     IE_NAME = 'viki:channel'
 351     _VALID_URL = r'%s(?:tv|news|movies|artists)/(?P<id>[0-9]+c)' % VikiBaseIE._VALID_URL_BASE
 352     _TESTS = [{
 353         'url': 'http://www.viki.com/tv/50c-boys-over-flowers',
 354         'info_dict': {
 355             'id': '50c',
 356             'title': 'Boys Over Flowers',
 357             'description': 'md5:ecd3cff47967fe193cff37c0bec52790',
 358         },
 359         'playlist_mincount': 71,
 360     }, {
 361         'url': 'http://www.viki.com/tv/1354c-poor-nastya-complete',
 362         'info_dict': {
 363             'id': '1354c',
 364             'title': 'Poor Nastya [COMPLETE]',
 365             'description': 'md5:05bf5471385aa8b21c18ad450e350525',
 366         },
 367         'playlist_count': 127,
 368     }, {
 369         'url': 'http://www.viki.com/news/24569c-showbiz-korea',
 370         'only_matching': True,
 371     }, {
 372         'url': 'http://www.viki.com/movies/22047c-pride-and-prejudice-2005',
 373         'only_matching': True,
 374     }, {
 375         'url': 'http://www.viki.com/artists/2141c-shinee',
 376         'only_matching': True,
 377     }]
 378
 379     _PER_PAGE = 25
 380
 381     def _real_extract(self, url):
 382         channel_id = self._match_id(url)
 383
 384         channel = self._call_api(
 385             'containers/%s.json' % channel_id, channel_id,
 386             'Downloading channel JSON')
 387
 388         self._check_errors(channel)
 389
 390         title = self.dict_selection(channel['titles'], 'en')
 391
 392         description = self.dict_selection(channel['descriptions'], 'en')
 393
 394         entries = []
 395         for video_type in ('episodes', 'clips', 'movies'):
 396             for page_num in itertools.count(1):
 397                 page = self._call_api(
 398                     'containers/%s/%s.json?per_page=%d&sort=number&direction=asc&with_paging=true&page=%d'
 399                     % (channel_id, video_type, self._PER_PAGE, page_num), channel_id,
 400                     'Downloading %s JSON page #%d' % (video_type, page_num))
 401                 for video in page['response']:
 402                     video_id = video['id']
 403                     entries.append(self.url_result(
 404                         'https://www.viki.com/videos/%s' % video_id, 'Viki'))
 405                 if not page['pagination']['next']:
 406                     break
 407
 408         return self.playlist_result(entries, channel_id, title, description)