yt_dlp/extractor/mildom.py

   1 # coding: utf-8
   2 from __future__ import unicode_literals
   3
   4 import base64
   5 from datetime import datetime
   6 import itertools
   7 import json
   8 import re
   9
  10 from .common import InfoExtractor
  11 from ..utils import (
  12     std_headers,
  13     update_url_query,
  14     random_uuidv4,
  15     try_get,
  16 )
  17 from ..compat import (
  18     compat_str,
  19 )
  20
  21
  22 class MildomBaseIE(InfoExtractor):
  23     _GUEST_ID = None
  24     _DISPATCHER_CONFIG = None
  25
  26     def _call_api(self, url, video_id, query={}, note='Downloading JSON metadata', init=False):
  27         url = update_url_query(url, self._common_queries(query, init=init))
  28         return self._download_json(url, video_id, note=note)['body']
  29
  30     def _common_queries(self, query={}, init=False):
  31         dc = self._fetch_dispatcher_config()
  32         r = {
  33             'timestamp': self.iso_timestamp(),
  34             '__guest_id': '' if init else self.guest_id(),
  35             '__location': dc['location'],
  36             '__country': dc['country'],
  37             '__cluster': dc['cluster'],
  38             '__platform': 'web',
  39             '__la': self.lang_code(),
  40             '__pcv': 'v2.9.44',
  41             'sfr': 'pc',
  42             'accessToken': '',
  43         }
  44         r.update(query)
  45         return r
  46
  47     def _fetch_dispatcher_config(self):
  48         if not self._DISPATCHER_CONFIG:
  49             tmp = self._download_json(
  50                 'https://disp.mildom.com/serverListV2', 'initialization',
  51                 note='Downloading dispatcher_config', data=json.dumps({
  52                     'protover': 0,
  53                     'data': base64.b64encode(json.dumps({
  54                         'fr': 'web',
  55                         'sfr': 'pc',
  56                         'devi': 'Windows',
  57                         'la': 'ja',
  58                         'gid': None,
  59                         'loc': '',
  60                         'clu': '',
  61                         'wh': '1919*810',
  62                         'rtm': self.iso_timestamp(),
  63                         'ua': std_headers['User-Agent'],
  64                     }).encode('utf8')).decode('utf8').replace('\n', ''),
  65                 }).encode('utf8'))
  66             self._DISPATCHER_CONFIG = self._parse_json(base64.b64decode(tmp['data']), 'initialization')
  67         return self._DISPATCHER_CONFIG
  68
  69     @staticmethod
  70     def iso_timestamp():
  71         'new Date().toISOString()'
  72         return datetime.utcnow().isoformat()[0:-3] + 'Z'
  73
  74     def guest_id(self):
  75         'getGuestId'
  76         if self._GUEST_ID:
  77             return self._GUEST_ID
  78         self._GUEST_ID = try_get(
  79             self, (
  80                 lambda x: x._call_api(
  81                     'https://cloudac.mildom.com/nonolive/gappserv/guest/h5init', 'initialization',
  82                     note='Downloading guest token', init=True)['guest_id'] or None,
  83                 lambda x: x._get_cookies('https://www.mildom.com').get('gid').value,
  84                 lambda x: x._get_cookies('https://m.mildom.com').get('gid').value,
  85             ), compat_str) or ''
  86         return self._GUEST_ID
  87
  88     def lang_code(self):
  89         'getCurrentLangCode'
  90         return 'ja'
  91
  92
  93 class MildomIE(MildomBaseIE):
  94     IE_NAME = 'mildom'
  95     IE_DESC = 'Record ongoing live by specific user in Mildom'
  96     _VALID_URL = r'https?://(?:(?:www|m)\.)mildom\.com/(?P<id>\d+)'
  97
  98     def _real_extract(self, url):
  99         video_id = self._match_id(url)
 100         url = 'https://www.mildom.com/%s' % video_id
 101
 102         webpage = self._download_webpage(url, video_id)
 103
 104         enterstudio = self._call_api(
 105             'https://cloudac.mildom.com/nonolive/gappserv/live/enterstudio', video_id,
 106             note='Downloading live metadata', query={'user_id': video_id})
 107         result_video_id = enterstudio.get('log_id', video_id)
 108
 109         title = try_get(
 110             enterstudio, (
 111                 lambda x: self._html_search_meta('twitter:description', webpage),
 112                 lambda x: x['anchor_intro'],
 113             ), compat_str)
 114         description = try_get(
 115             enterstudio, (
 116                 lambda x: x['intro'],
 117                 lambda x: x['live_intro'],
 118             ), compat_str)
 119         uploader = try_get(
 120             enterstudio, (
 121                 lambda x: self._html_search_meta('twitter:title', webpage),
 122                 lambda x: x['loginname'],
 123             ), compat_str)
 124
 125         servers = self._call_api(
 126             'https://cloudac.mildom.com/nonolive/gappserv/live/liveserver', result_video_id,
 127             note='Downloading live server list', query={
 128                 'user_id': video_id,
 129                 'live_server_type': 'hls',
 130             })
 131
 132         stream_query = self._common_queries({
 133             'streamReqId': random_uuidv4(),
 134             'is_lhls': '0',
 135         })
 136         m3u8_url = update_url_query(servers['stream_server'] + '/%s_master.m3u8' % video_id, stream_query)
 137         formats = self._extract_m3u8_formats(m3u8_url, result_video_id, 'mp4', headers={
 138             'Referer': 'https://www.mildom.com/',
 139             'Origin': 'https://www.mildom.com',
 140         }, note='Downloading m3u8 information')
 141
 142         del stream_query['streamReqId'], stream_query['timestamp']
 143         for fmt in formats:
 144             fmt.setdefault('http_headers', {})['Referer'] = 'https://www.mildom.com/'
 145
 146         self._sort_formats(formats)
 147
 148         return {
 149             'id': result_video_id,
 150             'title': title,
 151             'description': description,
 152             'uploader': uploader,
 153             'uploader_id': video_id,
 154             'formats': formats,
 155             'is_live': True,
 156         }
 157
 158
 159 class MildomVodIE(MildomBaseIE):
 160     IE_NAME = 'mildom:vod'
 161     IE_DESC = 'Download a VOD in Mildom'
 162     _VALID_URL = r'https?://(?:(?:www|m)\.)mildom\.com/playback/(?P<user_id>\d+)/(?P<id>(?P=user_id)-[a-zA-Z0-9]+)'
 163
 164     def _real_extract(self, url):
 165         m = re.match(self._VALID_URL, url)
 166         user_id, video_id = m.group('user_id'), m.group('id')
 167         url = 'https://www.mildom.com/playback/%s/%s' % (user_id, video_id)
 168
 169         webpage = self._download_webpage(url, video_id)
 170
 171         autoplay = self._call_api(
 172             'https://cloudac.mildom.com/nonolive/videocontent/playback/getPlaybackDetail', video_id,
 173             note='Downloading playback metadata', query={
 174                 'v_id': video_id,
 175             })['playback']
 176
 177         title = try_get(
 178             autoplay, (
 179                 lambda x: self._html_search_meta('og:description', webpage),
 180                 lambda x: x['title'],
 181             ), compat_str)
 182         description = try_get(
 183             autoplay, (
 184                 lambda x: x['video_intro'],
 185             ), compat_str)
 186         uploader = try_get(
 187             autoplay, (
 188                 lambda x: x['author_info']['login_name'],
 189             ), compat_str)
 190
 191         formats = [{
 192             'url': autoplay['audio_url'],
 193             'format_id': 'audio',
 194             'protocol': 'm3u8_native',
 195             'vcodec': 'none',
 196             'acodec': 'aac',
 197             'ext': 'm4a'
 198         }]
 199         for fmt in autoplay['video_link']:
 200             formats.append({
 201                 'format_id': 'video-%s' % fmt['name'],
 202                 'url': fmt['url'],
 203                 'protocol': 'm3u8_native',
 204                 'width': fmt['level'] * autoplay['video_width'] // autoplay['video_height'],
 205                 'height': fmt['level'],
 206                 'vcodec': 'h264',
 207                 'acodec': 'aac',
 208                 'ext': 'mp4'
 209             })
 210
 211         self._sort_formats(formats)
 212
 213         return {
 214             'id': video_id,
 215             'title': title,
 216             'description': description,
 217             'uploader': uploader,
 218             'uploader_id': user_id,
 219             'formats': formats,
 220         }
 221
 222
 223 class MildomUserVodIE(MildomBaseIE):
 224     IE_NAME = 'mildom:user:vod'
 225     IE_DESC = 'Download all VODs from specific user in Mildom'
 226     _VALID_URL = r'https?://(?:(?:www|m)\.)mildom\.com/profile/(?P<id>\d+)'
 227     _TESTS = [{
 228         'url': 'https://www.mildom.com/profile/10093333',
 229         'info_dict': {
 230             'id': '10093333',
 231             'title': 'Uploads from ねこばたけ',
 232         },
 233         'playlist_mincount': 351,
 234     }]
 235
 236     def _entries(self, user_id):
 237         for page in itertools.count(1):
 238             reply = self._call_api(
 239                 'https://cloudac.mildom.com/nonolive/videocontent/profile/playbackList',
 240                 user_id, note='Downloading page %d' % page, query={
 241                     'user_id': user_id,
 242                     'page': page,
 243                     'limit': '30',
 244                 })
 245             if not reply:
 246                 break
 247             for x in reply:
 248                 yield self.url_result('https://www.mildom.com/playback/%s/%s' % (user_id, x['v_id']))
 249
 250     def _real_extract(self, url):
 251         user_id = self._match_id(url)
 252         self.to_screen('This will download all VODs belonging to user. To download ongoing live video, use "https://www.mildom.com/%s" instead' % user_id)
 253
 254         profile = self._call_api(
 255             'https://cloudac.mildom.com/nonolive/gappserv/user/profileV2', user_id,
 256             query={'user_id': user_id}, note='Downloading user profile')['user_info']
 257
 258         return self.playlist_result(
 259             self._entries(user_id), user_id, 'Uploads from %s' % profile['loginname'])