[yt-dlp.git] / yt_dlp / extractor / mildom.py

# coding: utf-8
from __future__ import unicode_literals

import base64
from datetime import datetime
import itertools
import json

from .common import InfoExtractor
from ..utils import (
    std_headers,
    update_url_query,
    random_uuidv4,
    try_get,
)
from ..compat import (
    compat_str,
)


class MildomBaseIE(InfoExtractor):
    _GUEST_ID = None
    _DISPATCHER_CONFIG = None

    def _call_api(self, url, video_id, query={}, note='Downloading JSON metadata', init=False):
        url = update_url_query(url, self._common_queries(query, init=init))
        return self._download_json(url, video_id, note=note)['body']

    def _common_queries(self, query={}, init=False):
        dc = self._fetch_dispatcher_config()
        r = {
            'timestamp': self.iso_timestamp(),
            '__guest_id': '' if init else self.guest_id(),
            '__location': dc['location'],
            '__country': dc['country'],
            '__cluster': dc['cluster'],
            '__platform': 'web',
            '__la': self.lang_code(),
            '__pcv': 'v2.9.44',
            'sfr': 'pc',
            'accessToken': '',
        }
        r.update(query)
        return r

    def _fetch_dispatcher_config(self):
        if not self._DISPATCHER_CONFIG:
            tmp = self._download_json(
                'https://disp.mildom.com/serverListV2', 'initialization',
                note='Downloading dispatcher_config', data=json.dumps({
                    'protover': 0,
                    'data': base64.b64encode(json.dumps({
                        'fr': 'web',
                        'sfr': 'pc',
                        'devi': 'Windows',
                        'la': 'ja',
                        'gid': None,
                        'loc': '',
                        'clu': '',
                        'wh': '1919*810',
                        'rtm': self.iso_timestamp(),
                        'ua': std_headers['User-Agent'],
                    }).encode('utf8')).decode('utf8').replace('\n', ''),
                }).encode('utf8'))
            self._DISPATCHER_CONFIG = self._parse_json(base64.b64decode(tmp['data']), 'initialization')
        return self._DISPATCHER_CONFIG

    @staticmethod
    def iso_timestamp():
        'new Date().toISOString()'
        return datetime.utcnow().isoformat()[0:-3] + 'Z'

    def guest_id(self):
        'getGuestId'
        if self._GUEST_ID:
            return self._GUEST_ID
        self._GUEST_ID = try_get(
            self, (
                lambda x: x._call_api(
                    'https://cloudac.mildom.com/nonolive/gappserv/guest/h5init', 'initialization',
                    note='Downloading guest token', init=True)['guest_id'] or None,
                lambda x: x._get_cookies('https://www.mildom.com').get('gid').value,
                lambda x: x._get_cookies('https://m.mildom.com').get('gid').value,
            ), compat_str) or ''
        return self._GUEST_ID

    def lang_code(self):
        'getCurrentLangCode'
        return 'ja'


class MildomIE(MildomBaseIE):
    IE_NAME = 'mildom'
    IE_DESC = 'Record ongoing live by specific user in Mildom'
    _VALID_URL = r'https?://(?:(?:www|m)\.)mildom\.com/(?P<id>\d+)'

    def _real_extract(self, url):
        video_id = self._match_id(url)
        url = 'https://www.mildom.com/%s' % video_id

        webpage = self._download_webpage(url, video_id)

        enterstudio = self._call_api(
            'https://cloudac.mildom.com/nonolive/gappserv/live/enterstudio', video_id,
            note='Downloading live metadata', query={'user_id': video_id})
        result_video_id = enterstudio.get('log_id', video_id)

        title = try_get(
            enterstudio, (
                lambda x: self._html_search_meta('twitter:description', webpage),
                lambda x: x['anchor_intro'],
            ), compat_str)
        description = try_get(
            enterstudio, (
                lambda x: x['intro'],
                lambda x: x['live_intro'],
            ), compat_str)
        uploader = try_get(
            enterstudio, (
                lambda x: self._html_search_meta('twitter:title', webpage),
                lambda x: x['loginname'],
            ), compat_str)

        servers = self._call_api(
            'https://cloudac.mildom.com/nonolive/gappserv/live/liveserver', result_video_id,
            note='Downloading live server list', query={
                'user_id': video_id,
                'live_server_type': 'hls',
            })

        stream_query = self._common_queries({
            'streamReqId': random_uuidv4(),
            'is_lhls': '0',
        })
        m3u8_url = update_url_query(servers['stream_server'] + '/%s_master.m3u8' % video_id, stream_query)
        formats = self._extract_m3u8_formats(m3u8_url, result_video_id, 'mp4', headers={
            'Referer': 'https://www.mildom.com/',
            'Origin': 'https://www.mildom.com',
        }, note='Downloading m3u8 information')

        del stream_query['streamReqId'], stream_query['timestamp']
        for fmt in formats:
            fmt.setdefault('http_headers', {})['Referer'] = 'https://www.mildom.com/'

        self._sort_formats(formats)

        return {
            'id': result_video_id,
            'title': title,
            'description': description,
            'uploader': uploader,
            'uploader_id': video_id,
            'formats': formats,
            'is_live': True,
        }


class MildomVodIE(MildomBaseIE):
    IE_NAME = 'mildom:vod'
    IE_DESC = 'Download a VOD in Mildom'
    _VALID_URL = r'https?://(?:(?:www|m)\.)mildom\.com/playback/(?P<user_id>\d+)/(?P<id>(?P=user_id)-[a-zA-Z0-9]+)'

    def _real_extract(self, url):
        m = self._match_valid_url(url)
        user_id, video_id = m.group('user_id'), m.group('id')
        url = 'https://www.mildom.com/playback/%s/%s' % (user_id, video_id)

        webpage = self._download_webpage(url, video_id)

        autoplay = self._call_api(
            'https://cloudac.mildom.com/nonolive/videocontent/playback/getPlaybackDetail', video_id,
            note='Downloading playback metadata', query={
                'v_id': video_id,
            })['playback']

        title = try_get(
            autoplay, (
                lambda x: self._html_search_meta('og:description', webpage),
                lambda x: x['title'],
            ), compat_str)
        description = try_get(
            autoplay, (
                lambda x: x['video_intro'],
            ), compat_str)
        uploader = try_get(
            autoplay, (
                lambda x: x['author_info']['login_name'],
            ), compat_str)

        formats = [{
            'url': autoplay['audio_url'],
            'format_id': 'audio',
            'protocol': 'm3u8_native',
            'vcodec': 'none',
            'acodec': 'aac',
            'ext': 'm4a'
        }]
        for fmt in autoplay['video_link']:
            formats.append({
                'format_id': 'video-%s' % fmt['name'],
                'url': fmt['url'],
                'protocol': 'm3u8_native',
                'width': fmt['level'] * autoplay['video_width'] // autoplay['video_height'],
                'height': fmt['level'],
                'vcodec': 'h264',
                'acodec': 'aac',
                'ext': 'mp4'
            })

        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'uploader': uploader,
            'uploader_id': user_id,
            'formats': formats,
        }


class MildomUserVodIE(MildomBaseIE):
    IE_NAME = 'mildom:user:vod'
    IE_DESC = 'Download all VODs from specific user in Mildom'
    _VALID_URL = r'https?://(?:(?:www|m)\.)mildom\.com/profile/(?P<id>\d+)'
    _TESTS = [{
        'url': 'https://www.mildom.com/profile/10093333',
        'info_dict': {
            'id': '10093333',
            'title': 'Uploads from ねこばたけ',
        },
        'playlist_mincount': 351,
    }]

    def _entries(self, user_id):
        for page in itertools.count(1):
            reply = self._call_api(
                'https://cloudac.mildom.com/nonolive/videocontent/profile/playbackList',
                user_id, note='Downloading page %d' % page, query={
                    'user_id': user_id,
                    'page': page,
                    'limit': '30',
                })
            if not reply:
                break
            for x in reply:
                yield self.url_result('https://www.mildom.com/playback/%s/%s' % (user_id, x['v_id']))

    def _real_extract(self, url):
        user_id = self._match_id(url)
        self.to_screen('This will download all VODs belonging to user. To download ongoing live video, use "https://www.mildom.com/%s" instead' % user_id)

        profile = self._call_api(
            'https://cloudac.mildom.com/nonolive/gappserv/user/profileV2', user_id,
            query={'user_id': user_id}, note='Downloading user profile')['user_info']

        return self.playlist_result(
            self._entries(user_id), user_id, 'Uploads from %s' % profile['loginname'])
Commit	Line	Data
ffcb8191 THD	1	# coding: utf-8
	2	from __future__ import unicode_literals
	3
2cff4959	4	import base64
ffcb8191 THD	5	from datetime import datetime
	6	import itertools
	7	import json
ffcb8191 THD	8
	9	from .common import InfoExtractor
	10	from ..utils import (
a471f21d	11	std_headers,
ffcb8191 THD	12	update_url_query,
	13	random_uuidv4,
	14	try_get,
	15	)
	16	from ..compat import (
ffcb8191 THD	17	compat_str,
	18	)
	19
	20
	21	class MildomBaseIE(InfoExtractor):
	22	_GUEST_ID = None
	23	_DISPATCHER_CONFIG = None
	24
	25	def _call_api(self, url, video_id, query={}, note='Downloading JSON metadata', init=False):
	26	url = update_url_query(url, self._common_queries(query, init=init))
	27	return self._download_json(url, video_id, note=note)['body']
	28
	29	def _common_queries(self, query={}, init=False):
	30	dc = self._fetch_dispatcher_config()
	31	r = {
	32	'timestamp': self.iso_timestamp(),
	33	'__guest_id': '' if init else self.guest_id(),
	34	'__location': dc['location'],
	35	'__country': dc['country'],
	36	'__cluster': dc['cluster'],
	37	'__platform': 'web',
	38	'__la': self.lang_code(),
	39	'__pcv': 'v2.9.44',
	40	'sfr': 'pc',
	41	'accessToken': '',
	42	}
	43	r.update(query)
	44	return r
	45
	46	def _fetch_dispatcher_config(self):
	47	if not self._DISPATCHER_CONFIG:
a471f21d F	48	tmp = self._download_json(
	49	'https://disp.mildom.com/serverListV2', 'initialization',
	50	note='Downloading dispatcher_config', data=json.dumps({
	51	'protover': 0,
	52	'data': base64.b64encode(json.dumps({
	53	'fr': 'web',
	54	'sfr': 'pc',
	55	'devi': 'Windows',
	56	'la': 'ja',
	57	'gid': None,
	58	'loc': '',
	59	'clu': '',
	60	'wh': '1919*810',
	61	'rtm': self.iso_timestamp(),
	62	'ua': std_headers['User-Agent'],
	63	}).encode('utf8')).decode('utf8').replace('\n', ''),
	64	}).encode('utf8'))
	65	self._DISPATCHER_CONFIG = self._parse_json(base64.b64decode(tmp['data']), 'initialization')
ffcb8191 THD	66	return self._DISPATCHER_CONFIG
	67
	68	@staticmethod
	69	def iso_timestamp():
	70	'new Date().toISOString()'
	71	return datetime.utcnow().isoformat()[0:-3] + 'Z'
	72
	73	def guest_id(self):
	74	'getGuestId'
	75	if self._GUEST_ID:
	76	return self._GUEST_ID
	77	self._GUEST_ID = try_get(
	78	self, (
	79	lambda x: x._call_api(
	80	'https://cloudac.mildom.com/nonolive/gappserv/guest/h5init', 'initialization',
	81	note='Downloading guest token', init=True)['guest_id'] or None,
	82	lambda x: x._get_cookies('https://www.mildom.com').get('gid').value,
	83	lambda x: x._get_cookies('https://m.mildom.com').get('gid').value,
	84	), compat_str) or ''
	85	return self._GUEST_ID
	86
	87	def lang_code(self):
	88	'getCurrentLangCode'
	89	return 'ja'
	90
	91
	92	class MildomIE(MildomBaseIE):
	93	IE_NAME = 'mildom'
	94	IE_DESC = 'Record ongoing live by specific user in Mildom'
	95	_VALID_URL = r'https?://(?:(?:www\|m)\.)mildom\.com/(?P<id>\d+)'
	96
	97	def _real_extract(self, url):
	98	video_id = self._match_id(url)
	99	url = 'https://www.mildom.com/%s' % video_id
	100
	101	webpage = self._download_webpage(url, video_id)
	102
	103	enterstudio = self._call_api(
	104	'https://cloudac.mildom.com/nonolive/gappserv/live/enterstudio', video_id,
	105	note='Downloading live metadata', query={'user_id': video_id})
5d39972e	106	result_video_id = enterstudio.get('log_id', video_id)
ffcb8191 THD	107
	108	title = try_get(
	109	enterstudio, (
	110	lambda x: self._html_search_meta('twitter:description', webpage),
	111	lambda x: x['anchor_intro'],
	112	), compat_str)
	113	description = try_get(
	114	enterstudio, (
	115	lambda x: x['intro'],
	116	lambda x: x['live_intro'],
	117	), compat_str)
	118	uploader = try_get(
	119	enterstudio, (
	120	lambda x: self._html_search_meta('twitter:title', webpage),
	121	lambda x: x['loginname'],
	122	), compat_str)
	123
	124	servers = self._call_api(
5d39972e	125	'https://cloudac.mildom.com/nonolive/gappserv/live/liveserver', result_video_id,
ffcb8191 THD	126	note='Downloading live server list', query={
	127	'user_id': video_id,
	128	'live_server_type': 'hls',
	129	})
	130
	131	stream_query = self._common_queries({
	132	'streamReqId': random_uuidv4(),
	133	'is_lhls': '0',
	134	})
	135	m3u8_url = update_url_query(servers['stream_server'] + '/%s_master.m3u8' % video_id, stream_query)
5d39972e	136	formats = self._extract_m3u8_formats(m3u8_url, result_video_id, 'mp4', headers={
ffcb8191 THD	137	'Referer': 'https://www.mildom.com/',
	138	'Origin': 'https://www.mildom.com',
	139	}, note='Downloading m3u8 information')
2cff4959	140
ffcb8191 THD	141	del stream_query['streamReqId'], stream_query['timestamp']
ffcb8191 THD	142	for fmt in formats:
a471f21d	143	fmt.setdefault('http_headers', {})['Referer'] = 'https://www.mildom.com/'
ffcb8191 THD	144
	145	self._sort_formats(formats)
	146
	147	return {
5d39972e	148	'id': result_video_id,
ffcb8191 THD	149	'title': title,
	150	'description': description,
	151	'uploader': uploader,
	152	'uploader_id': video_id,
	153	'formats': formats,
	154	'is_live': True,
	155	}
	156
	157
	158	class MildomVodIE(MildomBaseIE):
	159	IE_NAME = 'mildom:vod'
	160	IE_DESC = 'Download a VOD in Mildom'
	161	_VALID_URL = r'https?://(?:(?:www\|m)\.)mildom\.com/playback/(?P<user_id>\d+)/(?P<id>(?P=user_id)-[a-zA-Z0-9]+)'
	162
	163	def _real_extract(self, url):
5ad28e7f	164	m = self._match_valid_url(url)
5d39972e	165	user_id, video_id = m.group('user_id'), m.group('id')
ffcb8191 THD	166	url = 'https://www.mildom.com/playback/%s/%s' % (user_id, video_id)
	167
	168	webpage = self._download_webpage(url, video_id)
	169
	170	autoplay = self._call_api(
	171	'https://cloudac.mildom.com/nonolive/videocontent/playback/getPlaybackDetail', video_id,
	172	note='Downloading playback metadata', query={
	173	'v_id': video_id,
	174	})['playback']
	175
	176	title = try_get(
	177	autoplay, (
	178	lambda x: self._html_search_meta('og:description', webpage),
	179	lambda x: x['title'],
	180	), compat_str)
	181	description = try_get(
	182	autoplay, (
	183	lambda x: x['video_intro'],
	184	), compat_str)
	185	uploader = try_get(
	186	autoplay, (
	187	lambda x: x['author_info']['login_name'],
	188	), compat_str)
	189
c1df120e	190	formats = [{
ffcb8191 THD	191	'url': autoplay['audio_url'],
	192	'format_id': 'audio',
	193	'protocol': 'm3u8_native',
	194	'vcodec': 'none',
	195	'acodec': 'aac',
c1df120e	196	'ext': 'm4a'
ffcb8191	197	}]
ffcb8191	198	for fmt in autoplay['video_link']:
c1df120e	199	formats.append({
ffcb8191 THD	200	'format_id': 'video-%s' % fmt['name'],
	201	'url': fmt['url'],
	202	'protocol': 'm3u8_native',
	203	'width': fmt['level'] * autoplay['video_width'] // autoplay['video_height'],
	204	'height': fmt['level'],
	205	'vcodec': 'h264',
	206	'acodec': 'aac',
c1df120e	207	'ext': 'mp4'
ffcb8191 THD	208	})
ffcb8191 THD	209
ffcb8191 THD	210	self._sort_formats(formats)
	211
	212	return {
	213	'id': video_id,
	214	'title': title,
	215	'description': description,
	216	'uploader': uploader,
	217	'uploader_id': user_id,
	218	'formats': formats,
	219	}
	220
	221
	222	class MildomUserVodIE(MildomBaseIE):
	223	IE_NAME = 'mildom:user:vod'
	224	IE_DESC = 'Download all VODs from specific user in Mildom'
	225	_VALID_URL = r'https?://(?:(?:www\|m)\.)mildom\.com/profile/(?P<id>\d+)'
	226	_TESTS = [{
	227	'url': 'https://www.mildom.com/profile/10093333',
	228	'info_dict': {
	229	'id': '10093333',
	230	'title': 'Uploads from ねこばたけ',
	231	},
	232	'playlist_mincount': 351,
	233	}]
	234
3097d9e5	235	def _entries(self, user_id):
ffcb8191 THD	236	for page in itertools.count(1):
	237	reply = self._call_api(
	238	'https://cloudac.mildom.com/nonolive/videocontent/profile/playbackList',
	239	user_id, note='Downloading page %d' % page, query={
	240	'user_id': user_id,
	241	'page': page,
	242	'limit': '30',
	243	})
	244	if not reply:
	245	break
3097d9e5	246	for x in reply:
	247	yield self.url_result('https://www.mildom.com/playback/%s/%s' % (user_id, x['v_id']))
	248
	249	def _real_extract(self, url):
	250	user_id = self._match_id(url)
	251	self.to_screen('This will download all VODs belonging to user. To download ongoing live video, use "https://www.mildom.com/%s" instead' % user_id)
	252
	253	profile = self._call_api(
	254	'https://cloudac.mildom.com/nonolive/gappserv/user/profileV2', user_id,
	255	query={'user_id': user_id}, note='Downloading user profile')['user_info']
	256
	257	return self.playlist_result(
	258	self._entries(user_id), user_id, 'Uploads from %s' % profile['loginname'])