yt_dlp/extractor/mildom.py

   1 import functools
   2 import json
   3 import uuid
   4
   5 from .common import InfoExtractor
   6 from ..utils import (
   7     ExtractorError,
   8     OnDemandPagedList,
   9     determine_ext,
  10     dict_get,
  11     float_or_none,
  12     traverse_obj,
  13 )
  14
  15
  16 class MildomBaseIE(InfoExtractor):
  17     _GUEST_ID = None
  18
  19     def _call_api(self, url, video_id, query=None, note='Downloading JSON metadata', body=None):
  20         if not self._GUEST_ID:
  21             self._GUEST_ID = f'pc-gp-{str(uuid.uuid4())}'
  22
  23         content = self._download_json(
  24             url, video_id, note=note, data=json.dumps(body).encode() if body else None,
  25             headers={'Content-Type': 'application/json'} if body else {},
  26             query={
  27                 '__guest_id': self._GUEST_ID,
  28                 '__platform': 'web',
  29                 **(query or {}),
  30             })
  31
  32         if content['code'] != 0:
  33             raise ExtractorError(
  34                 f'Mildom says: {content["message"]} (code {content["code"]})',
  35                 expected=True)
  36         return content['body']
  37
  38
  39 class MildomIE(MildomBaseIE):
  40     IE_NAME = 'mildom'
  41     IE_DESC = 'Record ongoing live by specific user in Mildom'
  42     _VALID_URL = r'https?://(?:(?:www|m)\.)mildom\.com/(?P<id>\d+)'
  43
  44     def _real_extract(self, url):
  45         video_id = self._match_id(url)
  46         webpage = self._download_webpage(f'https://www.mildom.com/{video_id}', video_id)
  47
  48         enterstudio = self._call_api(
  49             'https://cloudac.mildom.com/nonolive/gappserv/live/enterstudio', video_id,
  50             note='Downloading live metadata', query={'user_id': video_id})
  51         result_video_id = enterstudio.get('log_id', video_id)
  52
  53         servers = self._call_api(
  54             'https://cloudac.mildom.com/nonolive/gappserv/live/liveserver', result_video_id,
  55             note='Downloading live server list', query={
  56                 'user_id': video_id,
  57                 'live_server_type': 'hls',
  58             })
  59
  60         playback_token = self._call_api(
  61             'https://cloudac.mildom.com/nonolive/gappserv/live/token', result_video_id,
  62             note='Obtaining live playback token', body={'host_id': video_id, 'type': 'hls'})
  63         playback_token = traverse_obj(playback_token, ('data', ..., 'token'), get_all=False)
  64         if not playback_token:
  65             raise ExtractorError('Failed to obtain live playback token')
  66
  67         formats = self._extract_m3u8_formats(
  68             f'{servers["stream_server"]}/{video_id}_master.m3u8?{playback_token}',
  69             result_video_id, 'mp4', headers={
  70                 'Referer': 'https://www.mildom.com/',
  71                 'Origin': 'https://www.mildom.com',
  72             })
  73
  74         for fmt in formats:
  75             fmt.setdefault('http_headers', {})['Referer'] = 'https://www.mildom.com/'
  76
  77         return {
  78             'id': result_video_id,
  79             'title': self._html_search_meta('twitter:description', webpage, default=None) or traverse_obj(enterstudio, 'anchor_intro'),
  80             'description': traverse_obj(enterstudio, 'intro', 'live_intro', expected_type=str),
  81             'timestamp': float_or_none(enterstudio.get('live_start_ms'), scale=1000),
  82             'uploader': self._html_search_meta('twitter:title', webpage, default=None) or traverse_obj(enterstudio, 'loginname'),
  83             'uploader_id': video_id,
  84             'formats': formats,
  85             'is_live': True,
  86         }
  87
  88
  89 class MildomVodIE(MildomBaseIE):
  90     IE_NAME = 'mildom:vod'
  91     IE_DESC = 'VOD in Mildom'
  92     _VALID_URL = r'https?://(?:(?:www|m)\.)mildom\.com/playback/(?P<user_id>\d+)/(?P<id>(?P=user_id)-[a-zA-Z0-9]+-?[0-9]*)'
  93     _TESTS = [{
  94         'url': 'https://www.mildom.com/playback/10882672/10882672-1597662269',
  95         'info_dict': {
  96             'id': '10882672-1597662269',
  97             'ext': 'mp4',
  98             'title': '始めてのミルダム配信じゃぃ！',
  99             'thumbnail': r're:^https?://.*\.(png|jpg)$',
 100             'upload_date': '20200817',
 101             'duration': 4138.37,
 102             'description': 'ゲームをしたくて！',
 103             'timestamp': 1597662269.0,
 104             'uploader_id': '10882672',
 105             'uploader': 'kson組長(けいそん)',
 106         },
 107     }, {
 108         'url': 'https://www.mildom.com/playback/10882672/10882672-1597758589870-477',
 109         'info_dict': {
 110             'id': '10882672-1597758589870-477',
 111             'ext': 'mp4',
 112             'title': '【kson】感染メイズ！麻酔銃で無双する',
 113             'thumbnail': r're:^https?://.*\.(png|jpg)$',
 114             'timestamp': 1597759093.0,
 115             'uploader': 'kson組長(けいそん)',
 116             'duration': 4302.58,
 117             'uploader_id': '10882672',
 118             'description': 'このステージ絶対乗り越えたい',
 119             'upload_date': '20200818',
 120         },
 121     }, {
 122         'url': 'https://www.mildom.com/playback/10882672/10882672-buha9td2lrn97fk2jme0',
 123         'info_dict': {
 124             'id': '10882672-buha9td2lrn97fk2jme0',
 125             'ext': 'mp4',
 126             'title': '【kson組長】CART RACER!!!',
 127             'thumbnail': r're:^https?://.*\.(png|jpg)$',
 128             'uploader_id': '10882672',
 129             'uploader': 'kson組長(けいそん)',
 130             'upload_date': '20201104',
 131             'timestamp': 1604494797.0,
 132             'duration': 4657.25,
 133             'description': 'WTF',
 134         },
 135     }]
 136
 137     def _real_extract(self, url):
 138         user_id, video_id = self._match_valid_url(url).group('user_id', 'id')
 139         webpage = self._download_webpage(f'https://www.mildom.com/playback/{user_id}/{video_id}', video_id)
 140
 141         autoplay = self._call_api(
 142             'https://cloudac.mildom.com/nonolive/videocontent/playback/getPlaybackDetail', video_id,
 143             note='Downloading playback metadata', query={
 144                 'v_id': video_id,
 145             })['playback']
 146
 147         formats = [{
 148             'url': autoplay['audio_url'],
 149             'format_id': 'audio',
 150             'protocol': 'm3u8_native',
 151             'vcodec': 'none',
 152             'acodec': 'aac',
 153             'ext': 'm4a'
 154         }]
 155         for fmt in autoplay['video_link']:
 156             formats.append({
 157                 'format_id': 'video-%s' % fmt['name'],
 158                 'url': fmt['url'],
 159                 'protocol': 'm3u8_native',
 160                 'width': fmt['level'] * autoplay['video_width'] // autoplay['video_height'],
 161                 'height': fmt['level'],
 162                 'vcodec': 'h264',
 163                 'acodec': 'aac',
 164                 'ext': 'mp4'
 165             })
 166
 167         return {
 168             'id': video_id,
 169             'title': self._html_search_meta(('og:description', 'description'), webpage, default=None) or autoplay.get('title'),
 170             'description': traverse_obj(autoplay, 'video_intro'),
 171             'timestamp': float_or_none(autoplay.get('publish_time'), scale=1000),
 172             'duration': float_or_none(autoplay.get('video_length'), scale=1000),
 173             'thumbnail': dict_get(autoplay, ('upload_pic', 'video_pic')),
 174             'uploader': traverse_obj(autoplay, ('author_info', 'login_name')),
 175             'uploader_id': user_id,
 176             'formats': formats,
 177         }
 178
 179
 180 class MildomClipIE(MildomBaseIE):
 181     IE_NAME = 'mildom:clip'
 182     IE_DESC = 'Clip in Mildom'
 183     _VALID_URL = r'https?://(?:(?:www|m)\.)mildom\.com/clip/(?P<id>(?P<user_id>\d+)-[a-zA-Z0-9]+)'
 184     _TESTS = [{
 185         'url': 'https://www.mildom.com/clip/10042245-63921673e7b147ebb0806d42b5ba5ce9',
 186         'info_dict': {
 187             'id': '10042245-63921673e7b147ebb0806d42b5ba5ce9',
 188             'title': '全然違ったよ',
 189             'timestamp': 1619181890,
 190             'duration': 59,
 191             'thumbnail': r're:https?://.+',
 192             'uploader': 'ざきんぽ',
 193             'uploader_id': '10042245',
 194         },
 195     }, {
 196         'url': 'https://www.mildom.com/clip/10111524-ebf4036e5aa8411c99fb3a1ae0902864',
 197         'info_dict': {
 198             'id': '10111524-ebf4036e5aa8411c99fb3a1ae0902864',
 199             'title': 'かっこいい',
 200             'timestamp': 1621094003,
 201             'duration': 59,
 202             'thumbnail': r're:https?://.+',
 203             'uploader': '(ルーキー',
 204             'uploader_id': '10111524',
 205         },
 206     }, {
 207         'url': 'https://www.mildom.com/clip/10660174-2c539e6e277c4aaeb4b1fbe8d22cb902',
 208         'info_dict': {
 209             'id': '10660174-2c539e6e277c4aaeb4b1fbe8d22cb902',
 210             'title': 'あ',
 211             'timestamp': 1614769431,
 212             'duration': 31,
 213             'thumbnail': r're:https?://.+',
 214             'uploader': 'ドルゴルスレンギーン＝ダグワドルジ',
 215             'uploader_id': '10660174',
 216         },
 217     }]
 218
 219     def _real_extract(self, url):
 220         user_id, video_id = self._match_valid_url(url).group('user_id', 'id')
 221         webpage = self._download_webpage(f'https://www.mildom.com/clip/{video_id}', video_id)
 222
 223         clip_detail = self._call_api(
 224             'https://cloudac-cf-jp.mildom.com/nonolive/videocontent/clip/detail', video_id,
 225             note='Downloading playback metadata', query={
 226                 'clip_id': video_id,
 227             })
 228
 229         return {
 230             'id': video_id,
 231             'title': self._html_search_meta(
 232                 ('og:description', 'description'), webpage, default=None) or clip_detail.get('title'),
 233             'timestamp': float_or_none(clip_detail.get('create_time')),
 234             'duration': float_or_none(clip_detail.get('length')),
 235             'thumbnail': clip_detail.get('cover'),
 236             'uploader': traverse_obj(clip_detail, ('user_info', 'loginname')),
 237             'uploader_id': user_id,
 238
 239             'url': clip_detail['url'],
 240             'ext': determine_ext(clip_detail.get('url'), 'mp4'),
 241         }
 242
 243
 244 class MildomUserVodIE(MildomBaseIE):
 245     IE_NAME = 'mildom:user:vod'
 246     IE_DESC = 'Download all VODs from specific user in Mildom'
 247     _VALID_URL = r'https?://(?:(?:www|m)\.)mildom\.com/profile/(?P<id>\d+)'
 248     _TESTS = [{
 249         'url': 'https://www.mildom.com/profile/10093333',
 250         'info_dict': {
 251             'id': '10093333',
 252             'title': 'Uploads from ねこばたけ',
 253         },
 254         'playlist_mincount': 732,
 255     }, {
 256         'url': 'https://www.mildom.com/profile/10882672',
 257         'info_dict': {
 258             'id': '10882672',
 259             'title': 'Uploads from kson組長(けいそん)',
 260         },
 261         'playlist_mincount': 201,
 262     }]
 263
 264     def _fetch_page(self, user_id, page):
 265         page += 1
 266         reply = self._call_api(
 267             'https://cloudac.mildom.com/nonolive/videocontent/profile/playbackList',
 268             user_id, note=f'Downloading page {page}', query={
 269                 'user_id': user_id,
 270                 'page': page,
 271                 'limit': '30',
 272             })
 273         if not reply:
 274             return
 275         for x in reply:
 276             v_id = x.get('v_id')
 277             if not v_id:
 278                 continue
 279             yield self.url_result(f'https://www.mildom.com/playback/{user_id}/{v_id}')
 280
 281     def _real_extract(self, url):
 282         user_id = self._match_id(url)
 283         self.to_screen('This will download all VODs belonging to user. To download ongoing live video, use "https://www.mildom.com/%s" instead' % user_id)
 284
 285         profile = self._call_api(
 286             'https://cloudac.mildom.com/nonolive/gappserv/user/profileV2', user_id,
 287             query={'user_id': user_id}, note='Downloading user profile')['user_info']
 288
 289         return self.playlist_result(
 290             OnDemandPagedList(functools.partial(self._fetch_page, user_id), 30),
 291             user_id, f'Uploads from {profile["loginname"]}')