yt_dlp/extractor/vlive.py

   1 # coding: utf-8
   2 from __future__ import unicode_literals
   3
   4 import itertools
   5 import json
   6
   7 from .naver import NaverBaseIE
   8 from ..compat import (
   9     compat_HTTPError,
  10     compat_str,
  11 )
  12 from ..utils import (
  13     ExtractorError,
  14     int_or_none,
  15     merge_dicts,
  16     str_or_none,
  17     strip_or_none,
  18     try_get,
  19     urlencode_postdata,
  20 )
  21
  22
  23 class VLiveBaseIE(NaverBaseIE):
  24     _APP_ID = '8c6cc7b45d2568fb668be6e05b6e5a3b'
  25
  26
  27 class VLiveIE(VLiveBaseIE):
  28     IE_NAME = 'vlive'
  29     _VALID_URL = r'https?://(?:(?:www|m)\.)?vlive\.tv/(?:video|embed)/(?P<id>[0-9]+)'
  30     _NETRC_MACHINE = 'vlive'
  31     _TESTS = [{
  32         'url': 'http://www.vlive.tv/video/1326',
  33         'md5': 'cc7314812855ce56de70a06a27314983',
  34         'info_dict': {
  35             'id': '1326',
  36             'ext': 'mp4',
  37             'title': "Girl's Day's Broadcast",
  38             'creator': "Girl's Day",
  39             'view_count': int,
  40             'uploader_id': 'muploader_a',
  41         },
  42     }, {
  43         'url': 'http://www.vlive.tv/video/16937',
  44         'info_dict': {
  45             'id': '16937',
  46             'ext': 'mp4',
  47             'title': '첸백시 걍방',
  48             'creator': 'EXO',
  49             'view_count': int,
  50             'subtitles': 'mincount:12',
  51             'uploader_id': 'muploader_j',
  52         },
  53         'params': {
  54             'skip_download': True,
  55         },
  56     }, {
  57         'url': 'https://www.vlive.tv/video/129100',
  58         'md5': 'ca2569453b79d66e5b919e5d308bff6b',
  59         'info_dict': {
  60             'id': '129100',
  61             'ext': 'mp4',
  62             'title': '[V LIVE] [BTS+] Run BTS! 2019 - EP.71 :: Behind the scene',
  63             'creator': 'BTS+',
  64             'view_count': int,
  65             'subtitles': 'mincount:10',
  66         },
  67         'skip': 'This video is only available for CH+ subscribers',
  68     }, {
  69         'url': 'https://www.vlive.tv/embed/1326',
  70         'only_matching': True,
  71     }, {
  72         # works only with gcc=KR
  73         'url': 'https://www.vlive.tv/video/225019',
  74         'only_matching': True,
  75     }, {
  76         'url': 'https://www.vlive.tv/video/223906',
  77         'info_dict': {
  78             'id': '58',
  79             'title': 'RUN BTS!'
  80         },
  81         'playlist_mincount': 120
  82     }]
  83
  84     def _real_initialize(self):
  85         self._login()
  86
  87     def _login(self):
  88         email, password = self._get_login_info()
  89         if None in (email, password):
  90             return
  91
  92         def is_logged_in():
  93             login_info = self._download_json(
  94                 'https://www.vlive.tv/auth/loginInfo', None,
  95                 note='Downloading login info',
  96                 headers={'Referer': 'https://www.vlive.tv/home'})
  97             return try_get(
  98                 login_info, lambda x: x['message']['login'], bool) or False
  99
 100         LOGIN_URL = 'https://www.vlive.tv/auth/email/login'
 101         self._request_webpage(
 102             LOGIN_URL, None, note='Downloading login cookies')
 103
 104         self._download_webpage(
 105             LOGIN_URL, None, note='Logging in',
 106             data=urlencode_postdata({'email': email, 'pwd': password}),
 107             headers={
 108                 'Referer': LOGIN_URL,
 109                 'Content-Type': 'application/x-www-form-urlencoded'
 110             })
 111
 112         if not is_logged_in():
 113             raise ExtractorError('Unable to log in', expected=True)
 114
 115     def _call_api(self, path_template, video_id, fields=None, limit=None):
 116         query = {'appId': self._APP_ID, 'gcc': 'KR', 'platformType': 'PC'}
 117         if fields:
 118             query['fields'] = fields
 119         if limit:
 120             query['limit'] = limit
 121         try:
 122             return self._download_json(
 123                 'https://www.vlive.tv/globalv-web/vam-web/' + path_template % video_id, video_id,
 124                 'Downloading %s JSON metadata' % path_template.split('/')[-1].split('-')[0],
 125                 headers={'Referer': 'https://www.vlive.tv/'}, query=query)
 126         except ExtractorError as e:
 127             if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
 128                 self.raise_login_required(json.loads(e.cause.read().decode('utf-8'))['message'])
 129             raise
 130
 131     def _real_extract(self, url):
 132         video_id = self._match_id(url)
 133
 134         post = self._call_api(
 135             'post/v1.0/officialVideoPost-%s', video_id,
 136             'author{nickname},channel{channelCode,channelName},officialVideo{commentCount,exposeStatus,likeCount,playCount,playTime,status,title,type,vodId},playlist{playlistSeq,totalCount,name}')
 137
 138         playlist = post.get('playlist')
 139         if not playlist or self.get_param('noplaylist'):
 140             if playlist:
 141                 self.to_screen(
 142                     'Downloading just video %s because of --no-playlist'
 143                     % video_id)
 144
 145             video = post['officialVideo']
 146             return self._get_vlive_info(post, video, video_id)
 147         else:
 148             playlist_name = playlist.get('name')
 149             playlist_id = str_or_none(playlist.get('playlistSeq'))
 150             playlist_count = str_or_none(playlist.get('totalCount'))
 151
 152             playlist = self._call_api(
 153                 'playlist/v1.0/playlist-%s/posts', playlist_id, 'data', limit=playlist_count)
 154
 155             entries = []
 156             for video_data in playlist['data']:
 157                 video = video_data.get('officialVideo')
 158                 video_id = str_or_none(video.get('videoSeq'))
 159                 entries.append(self._get_vlive_info(video_data, video, video_id))
 160
 161             return self.playlist_result(entries, playlist_id, playlist_name)
 162
 163     def _get_vlive_info(self, post, video, video_id):
 164         def get_common_fields():
 165             channel = post.get('channel') or {}
 166             return {
 167                 'title': video.get('title'),
 168                 'creator': post.get('author', {}).get('nickname'),
 169                 'channel': channel.get('channelName'),
 170                 'channel_id': channel.get('channelCode'),
 171                 'duration': int_or_none(video.get('playTime')),
 172                 'view_count': int_or_none(video.get('playCount')),
 173                 'like_count': int_or_none(video.get('likeCount')),
 174                 'comment_count': int_or_none(video.get('commentCount')),
 175             }
 176
 177         video_type = video.get('type')
 178         if video_type == 'VOD':
 179             inkey = self._call_api('video/v1.0/vod/%s/inkey', video_id)['inkey']
 180             vod_id = video['vodId']
 181             info_dict = merge_dicts(
 182                 get_common_fields(),
 183                 self._extract_video_info(video_id, vod_id, inkey))
 184             thumbnail = video.get('thumb')
 185             if thumbnail:
 186                 if not info_dict.get('thumbnails') and info_dict.get('thumbnail'):
 187                     info_dict['thumbnails'] = [{'url': info_dict.pop('thumbnail')}]
 188                 info_dict.setdefault('thumbnails', []).append({'url': thumbnail, 'preference': 1})
 189             return info_dict
 190         elif video_type == 'LIVE':
 191             status = video.get('status')
 192             if status == 'ON_AIR':
 193                 stream_url = self._call_api(
 194                     'old/v3/live/%s/playInfo',
 195                     video_id)['result']['adaptiveStreamUrl']
 196                 formats = self._extract_m3u8_formats(stream_url, video_id, 'mp4')
 197                 self._sort_formats(formats)
 198                 info = get_common_fields()
 199                 info.update({
 200                     'title': self._live_title(video['title']),
 201                     'id': video_id,
 202                     'formats': formats,
 203                     'is_live': True,
 204                 })
 205                 return info
 206             elif status == 'ENDED':
 207                 raise ExtractorError(
 208                     'Uploading for replay. Please wait...', expected=True)
 209             elif status == 'RESERVED':
 210                 raise ExtractorError('Coming soon!', expected=True)
 211             elif video.get('exposeStatus') == 'CANCEL':
 212                 raise ExtractorError(
 213                     'We are sorry, but the live broadcast has been canceled.',
 214                     expected=True)
 215             else:
 216                 raise ExtractorError('Unknown status ' + status)
 217
 218
 219 class VLivePostIE(VLiveIE):
 220     IE_NAME = 'vlive:post'
 221     _VALID_URL = r'https?://(?:(?:www|m)\.)?vlive\.tv/post/(?P<id>\d-\d+)'
 222     _TESTS = [{
 223         # uploadType = SOS
 224         'url': 'https://www.vlive.tv/post/1-20088044',
 225         'info_dict': {
 226             'id': '1-20088044',
 227             'title': 'Hola estrellitas la tierra les dice hola (si era así no?) Ha...',
 228             'description': 'md5:fab8a1e50e6e51608907f46c7fa4b407',
 229         },
 230         'playlist_count': 3,
 231     }, {
 232         # uploadType = V
 233         'url': 'https://www.vlive.tv/post/1-20087926',
 234         'info_dict': {
 235             'id': '1-20087926',
 236             'title': 'James Corden: And so, the baby becamos the Papa💜😭💪😭',
 237         },
 238         'playlist_count': 1,
 239     }]
 240     _FVIDEO_TMPL = 'fvideo/v1.0/fvideo-%%s/%s'
 241     _SOS_TMPL = _FVIDEO_TMPL % 'sosPlayInfo'
 242     _INKEY_TMPL = _FVIDEO_TMPL % 'inKey'
 243
 244     def _real_extract(self, url):
 245         post_id = self._match_id(url)
 246
 247         post = self._call_api(
 248             'post/v1.0/post-%s', post_id,
 249             'attachments{video},officialVideo{videoSeq},plainBody,title')
 250
 251         video_seq = str_or_none(try_get(
 252             post, lambda x: x['officialVideo']['videoSeq']))
 253         if video_seq:
 254             return self.url_result(
 255                 'http://www.vlive.tv/video/' + video_seq,
 256                 VLiveIE.ie_key(), video_seq)
 257
 258         title = post['title']
 259         entries = []
 260         for idx, video in enumerate(post['attachments']['video'].values()):
 261             video_id = video.get('videoId')
 262             if not video_id:
 263                 continue
 264             upload_type = video.get('uploadType')
 265             upload_info = video.get('uploadInfo') or {}
 266             entry = None
 267             if upload_type == 'SOS':
 268                 download = self._call_api(
 269                     self._SOS_TMPL, video_id)['videoUrl']['download']
 270                 formats = []
 271                 for f_id, f_url in download.items():
 272                     formats.append({
 273                         'format_id': f_id,
 274                         'url': f_url,
 275                         'height': int_or_none(f_id[:-1]),
 276                     })
 277                 self._sort_formats(formats)
 278                 entry = {
 279                     'formats': formats,
 280                     'id': video_id,
 281                     'thumbnail': upload_info.get('imageUrl'),
 282                 }
 283             elif upload_type == 'V':
 284                 vod_id = upload_info.get('videoId')
 285                 if not vod_id:
 286                     continue
 287                 inkey = self._call_api(self._INKEY_TMPL, video_id)['inKey']
 288                 entry = self._extract_video_info(video_id, vod_id, inkey)
 289             if entry:
 290                 entry['title'] = '%s_part%s' % (title, idx)
 291                 entries.append(entry)
 292         return self.playlist_result(
 293             entries, post_id, title, strip_or_none(post.get('plainBody')))
 294
 295
 296 class VLiveChannelIE(VLiveBaseIE):
 297     IE_NAME = 'vlive:channel'
 298     _VALID_URL = r'https?://(?:channels\.vlive\.tv|(?:(?:www|m)\.)?vlive\.tv/channel)/(?P<id>[0-9A-Z]+)'
 299     _TESTS = [{
 300         'url': 'http://channels.vlive.tv/FCD4B',
 301         'info_dict': {
 302             'id': 'FCD4B',
 303             'title': 'MAMAMOO',
 304         },
 305         'playlist_mincount': 110
 306     }, {
 307         'url': 'https://www.vlive.tv/channel/FCD4B',
 308         'only_matching': True,
 309     }]
 310
 311     def _call_api(self, path, channel_key_suffix, channel_value, note, query):
 312         q = {
 313             'app_id': self._APP_ID,
 314             'channel' + channel_key_suffix: channel_value,
 315         }
 316         q.update(query)
 317         return self._download_json(
 318             'http://api.vfan.vlive.tv/vproxy/channelplus/' + path,
 319             channel_value, note='Downloading ' + note, query=q)['result']
 320
 321     def _real_extract(self, url):
 322         channel_code = self._match_id(url)
 323
 324         channel_seq = self._call_api(
 325             'decodeChannelCode', 'Code', channel_code,
 326             'decode channel code', {})['channelSeq']
 327
 328         channel_name = None
 329         entries = []
 330
 331         for page_num in itertools.count(1):
 332             video_list = self._call_api(
 333                 'getChannelVideoList', 'Seq', channel_seq,
 334                 'channel list page #%d' % page_num, {
 335                     # Large values of maxNumOfRows (~300 or above) may cause
 336                     # empty responses (see [1]), e.g. this happens for [2] that
 337                     # has more than 300 videos.
 338                     # 1. https://github.com/ytdl-org/youtube-dl/issues/13830
 339                     # 2. http://channels.vlive.tv/EDBF.
 340                     'maxNumOfRows': 100,
 341                     'pageNo': page_num
 342                 }
 343             )
 344
 345             if not channel_name:
 346                 channel_name = try_get(
 347                     video_list,
 348                     lambda x: x['channelInfo']['channelName'],
 349                     compat_str)
 350
 351             videos = try_get(
 352                 video_list, lambda x: x['videoList'], list)
 353             if not videos:
 354                 break
 355
 356             for video in videos:
 357                 video_id = video.get('videoSeq')
 358                 video_type = video.get('videoType')
 359
 360                 if not video_id or not video_type:
 361                     continue
 362                 video_id = compat_str(video_id)
 363
 364                 if video_type in ('PLAYLIST'):
 365                     first_video_id = try_get(
 366                         video,
 367                         lambda x: x['videoPlaylist']['videoList'][0]['videoSeq'], int)
 368
 369                     if not first_video_id:
 370                         continue
 371
 372                     entries.append(
 373                         self.url_result(
 374                             'http://www.vlive.tv/video/%s' % first_video_id,
 375                             ie=VLiveIE.ie_key(), video_id=first_video_id))
 376                 else:
 377                     entries.append(
 378                         self.url_result(
 379                             'http://www.vlive.tv/video/%s' % video_id,
 380                             ie=VLiveIE.ie_key(), video_id=video_id))
 381
 382         return self.playlist_result(
 383             entries, channel_code, channel_name)