yt_dlp/extractor/mildom.py

   1 # coding: utf-8
   2 from __future__ import unicode_literals
   3
   4 import base64
   5 from datetime import datetime
   6 import itertools
   7 import json
   8
   9 from .common import InfoExtractor
  10 from ..utils import (
  11     std_headers,
  12     update_url_query,
  13     random_uuidv4,
  14     try_get,
  15     float_or_none,
  16     dict_get
  17 )
  18 from ..compat import (
  19     compat_str,
  20 )
  21
  22
  23 class MildomBaseIE(InfoExtractor):
  24     _GUEST_ID = None
  25     _DISPATCHER_CONFIG = None
  26
  27     def _call_api(self, url, video_id, query=None, note='Downloading JSON metadata', init=False):
  28         query = query or {}
  29         if query:
  30             query['__platform'] = 'web'
  31         url = update_url_query(url, self._common_queries(query, init=init))
  32         content = self._download_json(url, video_id, note=note)
  33         if content['code'] == 0:
  34             return content['body']
  35         else:
  36             self.raise_no_formats(
  37                 f'Video not found or premium content. {content["code"]} - {content["message"]}',
  38                 expected=True)
  39
  40     def _common_queries(self, query={}, init=False):
  41         dc = self._fetch_dispatcher_config()
  42         r = {
  43             'timestamp': self.iso_timestamp(),
  44             '__guest_id': '' if init else self.guest_id(),
  45             '__location': dc['location'],
  46             '__country': dc['country'],
  47             '__cluster': dc['cluster'],
  48             '__platform': 'web',
  49             '__la': self.lang_code(),
  50             '__pcv': 'v2.9.44',
  51             'sfr': 'pc',
  52             'accessToken': '',
  53         }
  54         r.update(query)
  55         return r
  56
  57     def _fetch_dispatcher_config(self):
  58         if not self._DISPATCHER_CONFIG:
  59             tmp = self._download_json(
  60                 'https://disp.mildom.com/serverListV2', 'initialization',
  61                 note='Downloading dispatcher_config', data=json.dumps({
  62                     'protover': 0,
  63                     'data': base64.b64encode(json.dumps({
  64                         'fr': 'web',
  65                         'sfr': 'pc',
  66                         'devi': 'Windows',
  67                         'la': 'ja',
  68                         'gid': None,
  69                         'loc': '',
  70                         'clu': '',
  71                         'wh': '1919*810',
  72                         'rtm': self.iso_timestamp(),
  73                         'ua': std_headers['User-Agent'],
  74                     }).encode('utf8')).decode('utf8').replace('\n', ''),
  75                 }).encode('utf8'))
  76             self._DISPATCHER_CONFIG = self._parse_json(base64.b64decode(tmp['data']), 'initialization')
  77         return self._DISPATCHER_CONFIG
  78
  79     @staticmethod
  80     def iso_timestamp():
  81         'new Date().toISOString()'
  82         return datetime.utcnow().isoformat()[0:-3] + 'Z'
  83
  84     def guest_id(self):
  85         'getGuestId'
  86         if self._GUEST_ID:
  87             return self._GUEST_ID
  88         self._GUEST_ID = try_get(
  89             self, (
  90                 lambda x: x._call_api(
  91                     'https://cloudac.mildom.com/nonolive/gappserv/guest/h5init', 'initialization',
  92                     note='Downloading guest token', init=True)['guest_id'] or None,
  93                 lambda x: x._get_cookies('https://www.mildom.com').get('gid').value,
  94                 lambda x: x._get_cookies('https://m.mildom.com').get('gid').value,
  95             ), compat_str) or ''
  96         return self._GUEST_ID
  97
  98     def lang_code(self):
  99         'getCurrentLangCode'
 100         return 'ja'
 101
 102
 103 class MildomIE(MildomBaseIE):
 104     IE_NAME = 'mildom'
 105     IE_DESC = 'Record ongoing live by specific user in Mildom'
 106     _VALID_URL = r'https?://(?:(?:www|m)\.)mildom\.com/(?P<id>\d+)'
 107
 108     def _real_extract(self, url):
 109         video_id = self._match_id(url)
 110         url = 'https://www.mildom.com/%s' % video_id
 111
 112         webpage = self._download_webpage(url, video_id)
 113
 114         enterstudio = self._call_api(
 115             'https://cloudac.mildom.com/nonolive/gappserv/live/enterstudio', video_id,
 116             note='Downloading live metadata', query={'user_id': video_id})
 117         result_video_id = enterstudio.get('log_id', video_id)
 118
 119         title = try_get(
 120             enterstudio, (
 121                 lambda x: self._html_search_meta('twitter:description', webpage),
 122                 lambda x: x['anchor_intro'],
 123             ), compat_str)
 124         description = try_get(
 125             enterstudio, (
 126                 lambda x: x['intro'],
 127                 lambda x: x['live_intro'],
 128             ), compat_str)
 129         uploader = try_get(
 130             enterstudio, (
 131                 lambda x: self._html_search_meta('twitter:title', webpage),
 132                 lambda x: x['loginname'],
 133             ), compat_str)
 134
 135         servers = self._call_api(
 136             'https://cloudac.mildom.com/nonolive/gappserv/live/liveserver', result_video_id,
 137             note='Downloading live server list', query={
 138                 'user_id': video_id,
 139                 'live_server_type': 'hls',
 140             })
 141
 142         stream_query = self._common_queries({
 143             'streamReqId': random_uuidv4(),
 144             'is_lhls': '0',
 145         })
 146         m3u8_url = update_url_query(servers['stream_server'] + '/%s_master.m3u8' % video_id, stream_query)
 147         formats = self._extract_m3u8_formats(m3u8_url, result_video_id, 'mp4', headers={
 148             'Referer': 'https://www.mildom.com/',
 149             'Origin': 'https://www.mildom.com',
 150         }, note='Downloading m3u8 information')
 151
 152         del stream_query['streamReqId'], stream_query['timestamp']
 153         for fmt in formats:
 154             fmt.setdefault('http_headers', {})['Referer'] = 'https://www.mildom.com/'
 155
 156         self._sort_formats(formats)
 157
 158         return {
 159             'id': result_video_id,
 160             'title': title,
 161             'description': description,
 162             'timestamp': float_or_none(enterstudio.get('live_start_ms'), scale=1000),
 163             'uploader': uploader,
 164             'uploader_id': video_id,
 165             'formats': formats,
 166             'is_live': True,
 167         }
 168
 169
 170 class MildomVodIE(MildomBaseIE):
 171     IE_NAME = 'mildom:vod'
 172     IE_DESC = 'Download a VOD in Mildom'
 173     _VALID_URL = r'https?://(?:(?:www|m)\.)mildom\.com/playback/(?P<user_id>\d+)/(?P<id>(?P=user_id)-[a-zA-Z0-9]+-?[0-9]*)'
 174     _TESTS = [{
 175         'url': 'https://www.mildom.com/playback/10882672/10882672-1597662269',
 176         'info_dict': {
 177             'id': '10882672-1597662269',
 178             'ext': 'mp4',
 179             'title': '始めてのミルダム配信じゃぃ！',
 180             'thumbnail': r're:^https?://.*\.(png|jpg)$',
 181             'upload_date': '20200817',
 182             'duration': 4138.37,
 183             'description': 'ゲームをしたくて！',
 184             'timestamp': 1597662269.0,
 185             'uploader_id': '10882672',
 186             'uploader': 'kson組長(けいそん)',
 187         },
 188     }, {
 189         'url': 'https://www.mildom.com/playback/10882672/10882672-1597758589870-477',
 190         'info_dict': {
 191             'id': '10882672-1597758589870-477',
 192             'ext': 'mp4',
 193             'title': '【kson】感染メイズ！麻酔銃で無双する',
 194             'thumbnail': r're:^https?://.*\.(png|jpg)$',
 195             'timestamp': 1597759093.0,
 196             'uploader': 'kson組長(けいそん)',
 197             'duration': 4302.58,
 198             'uploader_id': '10882672',
 199             'description': 'このステージ絶対乗り越えたい',
 200             'upload_date': '20200818',
 201         },
 202     }, {
 203         'url': 'https://www.mildom.com/playback/10882672/10882672-buha9td2lrn97fk2jme0',
 204         'info_dict': {
 205             'id': '10882672-buha9td2lrn97fk2jme0',
 206             'ext': 'mp4',
 207             'title': '【kson組長】CART RACER!!!',
 208             'thumbnail': r're:^https?://.*\.(png|jpg)$',
 209             'uploader_id': '10882672',
 210             'uploader': 'kson組長(けいそん)',
 211             'upload_date': '20201104',
 212             'timestamp': 1604494797.0,
 213             'duration': 4657.25,
 214             'description': 'WTF',
 215         },
 216     }]
 217
 218     def _real_extract(self, url):
 219         m = self._match_valid_url(url)
 220         user_id, video_id = m.group('user_id'), m.group('id')
 221         url = 'https://www.mildom.com/playback/%s/%s' % (user_id, video_id)
 222
 223         webpage = self._download_webpage(url, video_id)
 224
 225         autoplay = self._call_api(
 226             'https://cloudac.mildom.com/nonolive/videocontent/playback/getPlaybackDetail', video_id,
 227             note='Downloading playback metadata', query={
 228                 'v_id': video_id,
 229             })['playback']
 230
 231         title = try_get(
 232             autoplay, (
 233                 lambda x: self._html_search_meta('og:description', webpage),
 234                 lambda x: x['title'],
 235             ), compat_str)
 236         description = try_get(
 237             autoplay, (
 238                 lambda x: x['video_intro'],
 239             ), compat_str)
 240         uploader = try_get(
 241             autoplay, (
 242                 lambda x: x['author_info']['login_name'],
 243             ), compat_str)
 244
 245         formats = [{
 246             'url': autoplay['audio_url'],
 247             'format_id': 'audio',
 248             'protocol': 'm3u8_native',
 249             'vcodec': 'none',
 250             'acodec': 'aac',
 251             'ext': 'm4a'
 252         }]
 253         for fmt in autoplay['video_link']:
 254             formats.append({
 255                 'format_id': 'video-%s' % fmt['name'],
 256                 'url': fmt['url'],
 257                 'protocol': 'm3u8_native',
 258                 'width': fmt['level'] * autoplay['video_width'] // autoplay['video_height'],
 259                 'height': fmt['level'],
 260                 'vcodec': 'h264',
 261                 'acodec': 'aac',
 262                 'ext': 'mp4'
 263             })
 264
 265         self._sort_formats(formats)
 266
 267         return {
 268             'id': video_id,
 269             'title': title,
 270             'description': description,
 271             'timestamp': float_or_none(autoplay['publish_time'], scale=1000),
 272             'duration': float_or_none(autoplay['video_length'], scale=1000),
 273             'thumbnail': dict_get(autoplay, ('upload_pic', 'video_pic')),
 274             'uploader': uploader,
 275             'uploader_id': user_id,
 276             'formats': formats,
 277         }
 278
 279
 280 class MildomUserVodIE(MildomBaseIE):
 281     IE_NAME = 'mildom:user:vod'
 282     IE_DESC = 'Download all VODs from specific user in Mildom'
 283     _VALID_URL = r'https?://(?:(?:www|m)\.)mildom\.com/profile/(?P<id>\d+)'
 284     _TESTS = [{
 285         'url': 'https://www.mildom.com/profile/10093333',
 286         'info_dict': {
 287             'id': '10093333',
 288             'title': 'Uploads from ねこばたけ',
 289         },
 290         'playlist_mincount': 351,
 291     }, {
 292         'url': 'https://www.mildom.com/profile/10882672',
 293         'info_dict': {
 294             'id': '10882672',
 295             'title': 'Uploads from kson組長(けいそん)',
 296         },
 297         'playlist_mincount': 191,
 298     }]
 299
 300     def _entries(self, user_id):
 301         for page in itertools.count(1):
 302             reply = self._call_api(
 303                 'https://cloudac.mildom.com/nonolive/videocontent/profile/playbackList',
 304                 user_id, note='Downloading page %d' % page, query={
 305                     'user_id': user_id,
 306                     'page': page,
 307                     'limit': '30',
 308                 })
 309             if not reply:
 310                 break
 311             for x in reply:
 312                 yield self.url_result('https://www.mildom.com/playback/%s/%s' % (user_id, x['v_id']))
 313
 314     def _real_extract(self, url):
 315         user_id = self._match_id(url)
 316         self.to_screen('This will download all VODs belonging to user. To download ongoing live video, use "https://www.mildom.com/%s" instead' % user_id)
 317
 318         profile = self._call_api(
 319             'https://cloudac.mildom.com/nonolive/gappserv/user/profileV2', user_id,
 320             query={'user_id': user_id}, note='Downloading user profile')['user_info']
 321
 322         return self.playlist_result(
 323             self._entries(user_id), user_id, 'Uploads from %s' % profile['loginname'])