[yt-dlp.git] / yt_dlp / extractor / fc2.py

import re

from .common import InfoExtractor
from ..compat import compat_parse_qs
from ..dependencies import websockets
from ..utils import (
    ExtractorError,
    WebSocketsWrapper,
    js_to_json,
    sanitized_Request,
    traverse_obj,
    update_url_query,
    urlencode_postdata,
    urljoin,
)


class FC2IE(InfoExtractor):
    _VALID_URL = r'^(?:https?://video\.fc2\.com/(?:[^/]+/)*content/|fc2:)(?P<id>[^/]+)'
    IE_NAME = 'fc2'
    _NETRC_MACHINE = 'fc2'
    _TESTS = [{
        'url': 'http://video.fc2.com/en/content/20121103kUan1KHs',
        'md5': 'a6ebe8ebe0396518689d963774a54eb7',
        'info_dict': {
            'id': '20121103kUan1KHs',
            'ext': 'flv',
            'title': 'Boxing again with Puff',
        },
    }, {
        'url': 'http://video.fc2.com/en/content/20150125cEva0hDn/',
        'info_dict': {
            'id': '20150125cEva0hDn',
            'ext': 'mp4',
        },
        'params': {
            'username': 'ytdl@yt-dl.org',
            'password': '(snip)',
        },
        'skip': 'requires actual password',
    }, {
        'url': 'http://video.fc2.com/en/a/content/20130926eZpARwsF',
        'only_matching': True,
    }]

    def _login(self):
        username, password = self._get_login_info()
        if username is None or password is None:
            return False

        # Log in
        login_form_strs = {
            'email': username,
            'password': password,
            'done': 'video',
            'Submit': ' Login ',
        }

        login_data = urlencode_postdata(login_form_strs)
        request = sanitized_Request(
            'https://secure.id.fc2.com/index.php?mode=login&switch_language=en', login_data)

        login_results = self._download_webpage(request, None, note='Logging in', errnote='Unable to log in')
        if 'mode=redirect&login=done' not in login_results:
            self.report_warning('unable to log in: bad username or password')
            return False

        # this is also needed
        login_redir = sanitized_Request('http://id.fc2.com/?mode=redirect&login=done')
        self._download_webpage(
            login_redir, None, note='Login redirect', errnote='Login redirect failed')

        return True

    def _real_extract(self, url):
        video_id = self._match_id(url)
        self._login()
        webpage = None
        if not url.startswith('fc2:'):
            webpage = self._download_webpage(url, video_id)
            self._downloader.cookiejar.clear_session_cookies()  # must clear
            self._login()

        title, thumbnail, description = None, None, None
        if webpage is not None:
            title = self._html_search_regex(
                (r'<h2\s+class="videoCnt_title">([^<]+?)</h2>',
                 r'\s+href="[^"]+"\s*title="([^"]+?)"\s*rel="nofollow">\s*<img',
                 # there's two matches in the webpage
                 r'\s+href="[^"]+"\s*title="([^"]+?)"\s*rel="nofollow">\s*\1'),
                webpage,
                'title', fatal=False)
            thumbnail = self._og_search_thumbnail(webpage)
            description = self._og_search_description(webpage, default=None)

        vidplaylist = self._download_json(
            'https://video.fc2.com/api/v3/videoplaylist/%s?sh=1&fs=0' % video_id, video_id,
            note='Downloading info page')
        vid_url = traverse_obj(vidplaylist, ('playlist', 'nq'))
        if not vid_url:
            raise ExtractorError('Unable to extract video URL')
        vid_url = urljoin('https://video.fc2.com/', vid_url)

        return {
            'id': video_id,
            'title': title,
            'url': vid_url,
            'ext': 'mp4',
            'protocol': 'm3u8_native',
            'description': description,
            'thumbnail': thumbnail,
        }


class FC2EmbedIE(InfoExtractor):
    _VALID_URL = r'https?://video\.fc2\.com/flv2\.swf\?(?P<query>.+)'
    IE_NAME = 'fc2:embed'

    _TEST = {
        'url': 'http://video.fc2.com/flv2.swf?t=201404182936758512407645&i=20130316kwishtfitaknmcgd76kjd864hso93htfjcnaogz629mcgfs6rbfk0hsycma7shkf85937cbchfygd74&i=201403223kCqB3Ez&d=2625&sj=11&lang=ja&rel=1&from=11&cmt=1&tk=TlRBM09EQTNNekU9&tl=プリズン･ブレイク%20S1-01%20マイケル%20【吹替】',
        'md5': 'b8aae5334cb691bdb1193a88a6ab5d5a',
        'info_dict': {
            'id': '201403223kCqB3Ez',
            'ext': 'flv',
            'title': 'プリズン･ブレイク S1-01 マイケル 【吹替】',
            'thumbnail': r're:^https?://.*\.jpg$',
        },
    }

    def _real_extract(self, url):
        mobj = self._match_valid_url(url)
        query = compat_parse_qs(mobj.group('query'))

        video_id = query['i'][-1]
        title = query.get('tl', ['FC2 video %s' % video_id])[0]

        sj = query.get('sj', [None])[0]
        thumbnail = None
        if sj:
            # See thumbnailImagePath() in ServerConst.as of flv2.swf
            thumbnail = 'http://video%s-thumbnail.fc2.com/up/pic/%s.jpg' % (
                sj, '/'.join((video_id[:6], video_id[6:8], video_id[-2], video_id[-1], video_id)))

        return {
            '_type': 'url_transparent',
            'ie_key': FC2IE.ie_key(),
            'url': 'fc2:%s' % video_id,
            'title': title,
            'thumbnail': thumbnail,
        }


class FC2LiveIE(InfoExtractor):
    _VALID_URL = r'https?://live\.fc2\.com/(?P<id>\d+)'
    IE_NAME = 'fc2:live'

    _TESTS = [{
        'url': 'https://live.fc2.com/57892267/',
        'info_dict': {
            'id': '57892267',
            'title': 'どこまで・・・',
            'uploader': 'あつあげ',
            'uploader_id': '57892267',
            'thumbnail': r're:https?://.+fc2.+',
        },
        'skip': 'livestream',
    }]

    def _real_extract(self, url):
        if not websockets:
            raise ExtractorError('websockets library is not available. Please install it.', expected=True)
        video_id = self._match_id(url)
        webpage = self._download_webpage('https://live.fc2.com/%s/' % video_id, video_id)

        self._set_cookie('live.fc2.com', 'js-player_size', '1')

        member_api = self._download_json(
            'https://live.fc2.com/api/memberApi.php', video_id, data=urlencode_postdata({
                'channel': '1',
                'profile': '1',
                'user': '1',
                'streamid': video_id
            }), note='Requesting member info')

        control_server = self._download_json(
            'https://live.fc2.com/api/getControlServer.php', video_id, note='Downloading ControlServer data',
            data=urlencode_postdata({
                'channel_id': video_id,
                'mode': 'play',
                'orz': '',
                'channel_version': member_api['data']['channel_data']['version'],
                'client_version': '2.1.0\n [1]',
                'client_type': 'pc',
                'client_app': 'browser_hls',
                'ipv6': '',
            }), headers={'X-Requested-With': 'XMLHttpRequest'})
        self._set_cookie('live.fc2.com', 'l_ortkn', control_server['orz_raw'])

        ws_url = update_url_query(control_server['url'], {'control_token': control_server['control_token']})
        playlist_data = None

        self.to_screen('%s: Fetching HLS playlist info via WebSocket' % video_id)
        ws = WebSocketsWrapper(ws_url, {
            'Cookie': str(self._get_cookies('https://live.fc2.com/'))[12:],
            'Origin': 'https://live.fc2.com',
            'Accept': '*/*',
            'User-Agent': self.get_param('http_headers')['User-Agent'],
        })

        self.write_debug('Sending HLS server request')

        while True:
            recv = ws.recv()
            if not recv:
                continue
            data = self._parse_json(recv, video_id, fatal=False)
            if not data or not isinstance(data, dict):
                continue

            if data.get('name') == 'connect_complete':
                break
        ws.send(r'{"name":"get_hls_information","arguments":{},"id":1}')

        while True:
            recv = ws.recv()
            if not recv:
                continue
            data = self._parse_json(recv, video_id, fatal=False)
            if not data or not isinstance(data, dict):
                continue
            if data.get('name') == '_response_' and data.get('id') == 1:
                self.write_debug('Goodbye')
                playlist_data = data
                break
            self.write_debug('Server said: %s%s' % (recv[:100], '...' if len(recv) > 100 else ''))

        if not playlist_data:
            raise ExtractorError('Unable to fetch HLS playlist info via WebSocket')

        formats = []
        for name, playlists in playlist_data['arguments'].items():
            if not isinstance(playlists, list):
                continue
            for pl in playlists:
                if pl.get('status') == 0 and 'master_playlist' in pl.get('url'):
                    formats.extend(self._extract_m3u8_formats(
                        pl['url'], video_id, ext='mp4', m3u8_id=name, live=True,
                        headers={
                            'Origin': 'https://live.fc2.com',
                            'Referer': url,
                        }))

        self._sort_formats(formats)
        for fmt in formats:
            fmt.update({
                'protocol': 'fc2_live',
                'ws': ws,
            })

        title = self._html_search_meta(('og:title', 'twitter:title'), webpage, 'live title', fatal=False)
        if not title:
            title = self._html_extract_title(webpage, 'html title', fatal=False)
            if title:
                # remove service name in <title>
                title = re.sub(r'\s+-\s+.+$', '', title)
        uploader = None
        if title:
            match = self._search_regex(r'^(.+?)\s*\[(.+?)\]$', title, 'title and uploader', default=None, group=(1, 2))
            if match and all(match):
                title, uploader = match

        live_info_view = self._search_regex(r'(?s)liveInfoView\s*:\s*({.+?}),\s*premiumStateView', webpage, 'user info', fatal=False) or None
        if live_info_view:
            # remove jQuery code from object literal
            live_info_view = re.sub(r'\$\(.+?\)[^,]+,', '"",', live_info_view)
            live_info_view = self._parse_json(js_to_json(live_info_view), video_id)

        return {
            'id': video_id,
            'title': title or traverse_obj(live_info_view, 'title'),
            'description': self._html_search_meta(
                ('og:description', 'twitter:description'),
                webpage, 'live description', fatal=False) or traverse_obj(live_info_view, 'info'),
            'formats': formats,
            'uploader': uploader or traverse_obj(live_info_view, 'name'),
            'uploader_id': video_id,
            'thumbnail': traverse_obj(live_info_view, 'thumb'),
            'is_live': True,
        }
Commit	Line	Data
15dfb392 LNO	1	import re
15dfb392 LNO	2
8e71456a	3	from .common import InfoExtractor
8a82af35	4	from ..compat import compat_parse_qs
9b8ee23b	5	from ..dependencies import websockets
1cc79574 PH	6	from ..utils import (
1cc79574 PH	7	ExtractorError,
15dfb392	8	WebSocketsWrapper,
15dfb392	9	js_to_json,
5c2266df	10	sanitized_Request,
d6bc443b	11	traverse_obj,
15dfb392	12	update_url_query,
6e6bc8da	13	urlencode_postdata,
d6bc443b	14	urljoin,
1cc79574	15	)
8e71456a PH	16
	17
	18	class FC2IE(InfoExtractor):
cf0efe96	19	_VALID_URL = r'^(?:https?://video\.fc2\.com/(?:[^/]+/)*content/\|fc2:)(?P<id>[^/]+)'
8e71456a	20	IE_NAME = 'fc2'
4231235c	21	_NETRC_MACHINE = 'fc2'
8940b860	22	_TESTS = [{
8e71456a PH	23	'url': 'http://video.fc2.com/en/content/20121103kUan1KHs',
	24	'md5': 'a6ebe8ebe0396518689d963774a54eb7',
	25	'info_dict': {
	26	'id': '20121103kUan1KHs',
	27	'ext': 'flv',
	28	'title': 'Boxing again with Puff',
	29	},
8940b860 PH	30	}, {
	31	'url': 'http://video.fc2.com/en/content/20150125cEva0hDn/',
	32	'info_dict': {
	33	'id': '20150125cEva0hDn',
	34	'ext': 'mp4',
	35	},
	36	'params': {
	37	'username': 'ytdl@yt-dl.org',
	38	'password': '(snip)',
38d05d17 YCH	39	},
38d05d17 YCH	40	'skip': 'requires actual password',
52dfb7ff S	41	}, {
	42	'url': 'http://video.fc2.com/en/a/content/20130926eZpARwsF',
	43	'only_matching': True,
8940b860	44	}]
4231235c	45
4231235c	46	def _login(self):
68217024	47	username, password = self._get_login_info()
8940b860 PH	48	if username is None or password is None:
8940b860 PH	49	return False
4231235c	50
	51	# Log in
	52	login_form_strs = {
8940b860	53	'email': username,
4231235c	54	'password': password,
8940b860 PH	55	'done': 'video',
8940b860 PH	56	'Submit': ' Login ',
4231235c	57	}
4231235c	58
6e6bc8da	59	login_data = urlencode_postdata(login_form_strs)
5c2266df	60	request = sanitized_Request(
4231235c	61	'https://secure.id.fc2.com/index.php?mode=login&switch_language=en', login_data)
	62
	63	login_results = self._download_webpage(request, None, note='Logging in', errnote='Unable to log in')
	64	if 'mode=redirect&login=done' not in login_results:
8940b860	65	self.report_warning('unable to log in: bad username or password')
4231235c	66	return False
8940b860	67
4231235c	68	# this is also needed
5c2266df	69	login_redir = sanitized_Request('http://id.fc2.com/?mode=redirect&login=done')
8940b860 PH	70	self._download_webpage(
8940b860 PH	71	login_redir, None, note='Login redirect', errnote='Login redirect failed')
4231235c	72
4231235c	73	return True
8e71456a PH	74
8e71456a PH	75	def _real_extract(self, url):
1cc79574	76	video_id = self._match_id(url)
8940b860	77	self._login()
cf0efe96 YCH	78	webpage = None
	79	if not url.startswith('fc2:'):
	80	webpage = self._download_webpage(url, video_id)
	81	self._downloader.cookiejar.clear_session_cookies() # must clear
	82	self._login()
	83
d6bc443b	84	title, thumbnail, description = None, None, None
cf0efe96	85	if webpage is not None:
d6bc443b LNO	86	title = self._html_search_regex(
	87	(r'<h2\s+class="videoCnt_title">([^<]+?)</h2>',
	88	r'\s+href="[^"]+"\stitle="([^"]+?)"\srel="nofollow">\s*<img',
	89	# there's two matches in the webpage
	90	r'\s+href="[^"]+"\stitle="([^"]+?)"\srel="nofollow">\s*\1'),
	91	webpage,
	92	'title', fatal=False)
cf0efe96	93	thumbnail = self._og_search_thumbnail(webpage)
81c5f44c	94	description = self._og_search_description(webpage, default=None)
8e71456a	95
d6bc443b LNO	96	vidplaylist = self._download_json(
	97	'https://video.fc2.com/api/v3/videoplaylist/%s?sh=1&fs=0' % video_id, video_id,
	98	note='Downloading info page')
	99	vid_url = traverse_obj(vidplaylist, ('playlist', 'nq'))
	100	if not vid_url:
	101	raise ExtractorError('Unable to extract video URL')
	102	vid_url = urljoin('https://video.fc2.com/', vid_url)
8e71456a PH	103
	104	return {
	105	'id': video_id,
23ae281b	106	'title': title,
d6bc443b LNO	107	'url': vid_url,
d6bc443b LNO	108	'ext': 'mp4',
81c5f44c	109	'protocol': 'm3u8_native',
d6bc443b	110	'description': description,
8e71456a PH	111	'thumbnail': thumbnail,
8e71456a PH	112	}
cf0efe96 YCH	113
	114
	115	class FC2EmbedIE(InfoExtractor):
	116	_VALID_URL = r'https?://video\.fc2\.com/flv2\.swf\?(?P<query>.+)'
	117	IE_NAME = 'fc2:embed'
	118
	119	_TEST = {
	120	'url': 'http://video.fc2.com/flv2.swf?t=201404182936758512407645&i=20130316kwishtfitaknmcgd76kjd864hso93htfjcnaogz629mcgfs6rbfk0hsycma7shkf85937cbchfygd74&i=201403223kCqB3Ez&d=2625&sj=11&lang=ja&rel=1&from=11&cmt=1&tk=TlRBM09EQTNNekU9&tl=プリズン･ブレイク%20S1-01%20マイケル%20【吹替】',
	121	'md5': 'b8aae5334cb691bdb1193a88a6ab5d5a',
	122	'info_dict': {
	123	'id': '201403223kCqB3Ez',
	124	'ext': 'flv',
	125	'title': 'プリズン･ブレイク S1-01 マイケル【吹替】',
ec85ded8	126	'thumbnail': r're:^https?://.*\.jpg$',
cf0efe96 YCH	127	},
	128	}
	129
	130	def _real_extract(self, url):
5ad28e7f	131	mobj = self._match_valid_url(url)
cf0efe96 YCH	132	query = compat_parse_qs(mobj.group('query'))
	133
	134	video_id = query['i'][-1]
	135	title = query.get('tl', ['FC2 video %s' % video_id])[0]
	136
	137	sj = query.get('sj', [None])[0]
	138	thumbnail = None
	139	if sj:
	140	# See thumbnailImagePath() in ServerConst.as of flv2.swf
	141	thumbnail = 'http://video%s-thumbnail.fc2.com/up/pic/%s.jpg' % (
	142	sj, '/'.join((video_id[:6], video_id[6:8], video_id[-2], video_id[-1], video_id)))
	143
	144	return {
	145	'_type': 'url_transparent',
ed2bfe93	146	'ie_key': FC2IE.ie_key(),
cf0efe96 YCH	147	'url': 'fc2:%s' % video_id,
	148	'title': title,
	149	'thumbnail': thumbnail,
	150	}
15dfb392 LNO	151
	152
	153	class FC2LiveIE(InfoExtractor):
	154	_VALID_URL = r'https?://live\.fc2\.com/(?P<id>\d+)'
	155	IE_NAME = 'fc2:live'
	156
	157	_TESTS = [{
	158	'url': 'https://live.fc2.com/57892267/',
	159	'info_dict': {
	160	'id': '57892267',
	161	'title': 'どこまで・・・',
	162	'uploader': 'あつあげ',
	163	'uploader_id': '57892267',
	164	'thumbnail': r're:https?://.+fc2.+',
	165	},
	166	'skip': 'livestream',
	167	}]
	168
	169	def _real_extract(self, url):
9b8ee23b	170	if not websockets:
15dfb392 LNO	171	raise ExtractorError('websockets library is not available. Please install it.', expected=True)
	172	video_id = self._match_id(url)
	173	webpage = self._download_webpage('https://live.fc2.com/%s/' % video_id, video_id)
	174
	175	self._set_cookie('live.fc2.com', 'js-player_size', '1')
	176
	177	member_api = self._download_json(
	178	'https://live.fc2.com/api/memberApi.php', video_id, data=urlencode_postdata({
	179	'channel': '1',
	180	'profile': '1',
	181	'user': '1',
	182	'streamid': video_id
	183	}), note='Requesting member info')
	184
	185	control_server = self._download_json(
	186	'https://live.fc2.com/api/getControlServer.php', video_id, note='Downloading ControlServer data',
	187	data=urlencode_postdata({
	188	'channel_id': video_id,
	189	'mode': 'play',
	190	'orz': '',
	191	'channel_version': member_api['data']['channel_data']['version'],
	192	'client_version': '2.1.0\n [1]',
	193	'client_type': 'pc',
	194	'client_app': 'browser_hls',
	195	'ipv6': '',
	196	}), headers={'X-Requested-With': 'XMLHttpRequest'})
	197	self._set_cookie('live.fc2.com', 'l_ortkn', control_server['orz_raw'])
	198
	199	ws_url = update_url_query(control_server['url'], {'control_token': control_server['control_token']})
	200	playlist_data = None
	201
	202	self.to_screen('%s: Fetching HLS playlist info via WebSocket' % video_id)
	203	ws = WebSocketsWrapper(ws_url, {
	204	'Cookie': str(self._get_cookies('https://live.fc2.com/'))[12:],
	205	'Origin': 'https://live.fc2.com',
	206	'Accept': '/',
21633673	207	'User-Agent': self.get_param('http_headers')['User-Agent'],
15dfb392	208	})
15dfb392	209
8a82af35	210	self.write_debug('Sending HLS server request')
15dfb392 LNO	211
	212	while True:
	213	recv = ws.recv()
	214	if not recv:
	215	continue
	216	data = self._parse_json(recv, video_id, fatal=False)
	217	if not data or not isinstance(data, dict):
	218	continue
	219
	220	if data.get('name') == 'connect_complete':
	221	break
	222	ws.send(r'{"name":"get_hls_information","arguments":{},"id":1}')
	223
	224	while True:
	225	recv = ws.recv()
	226	if not recv:
	227	continue
	228	data = self._parse_json(recv, video_id, fatal=False)
	229	if not data or not isinstance(data, dict):
	230	continue
	231	if data.get('name') == '_response_' and data.get('id') == 1:
8a82af35	232	self.write_debug('Goodbye')
15dfb392 LNO	233	playlist_data = data
15dfb392 LNO	234	break
8a82af35	235	self.write_debug('Server said: %s%s' % (recv[:100], '...' if len(recv) > 100 else ''))
15dfb392 LNO	236
	237	if not playlist_data:
	238	raise ExtractorError('Unable to fetch HLS playlist info via WebSocket')
	239
	240	formats = []
	241	for name, playlists in playlist_data['arguments'].items():
	242	if not isinstance(playlists, list):
	243	continue
	244	for pl in playlists:
	245	if pl.get('status') == 0 and 'master_playlist' in pl.get('url'):
	246	formats.extend(self._extract_m3u8_formats(
	247	pl['url'], video_id, ext='mp4', m3u8_id=name, live=True,
	248	headers={
	249	'Origin': 'https://live.fc2.com',
	250	'Referer': url,
	251	}))
	252
	253	self._sort_formats(formats)
	254	for fmt in formats:
	255	fmt.update({
	256	'protocol': 'fc2_live',
	257	'ws': ws,
	258	})
	259
	260	title = self._html_search_meta(('og:title', 'twitter:title'), webpage, 'live title', fatal=False)
	261	if not title:
	262	title = self._html_extract_title(webpage, 'html title', fatal=False)
	263	if title:
	264	# remove service name in <title>
	265	title = re.sub(r'\s+-\s+.+$', '', title)
	266	uploader = None
	267	if title:
	268	match = self._search_regex(r'^(.+?)\s*\[(.+?)\]$', title, 'title and uploader', default=None, group=(1, 2))
	269	if match and all(match):
	270	title, uploader = match
	271
	272	live_info_view = self._search_regex(r'(?s)liveInfoView\s:\s({.+?}),\s*premiumStateView', webpage, 'user info', fatal=False) or None
	273	if live_info_view:
	274	# remove jQuery code from object literal
	275	live_info_view = re.sub(r'\$\(.+?\)[^,]+,', '"",', live_info_view)
	276	live_info_view = self._parse_json(js_to_json(live_info_view), video_id)
	277
	278	return {
	279	'id': video_id,
	280	'title': title or traverse_obj(live_info_view, 'title'),
	281	'description': self._html_search_meta(
	282	('og:description', 'twitter:description'),
	283	webpage, 'live description', fatal=False) or traverse_obj(live_info_view, 'info'),
	284	'formats': formats,
	285	'uploader': uploader or traverse_obj(live_info_view, 'name'),
	286	'uploader_id': video_id,
	287	'thumbnail': traverse_obj(live_info_view, 'thumb'),
	288	'is_live': True,
	289	}