[yt-dlp.git] / yt_dlp / extractor / fc2.py

import re

from .common import InfoExtractor
from ..compat import compat_parse_qs
from ..networking import Request
from ..utils import (
    ExtractorError,
    js_to_json,
    traverse_obj,
    update_url_query,
    urlencode_postdata,
    urljoin,
)


class FC2IE(InfoExtractor):
    _VALID_URL = r'^(?:https?://video\.fc2\.com/(?:[^/]+/)*content/|fc2:)(?P<id>[^/]+)'
    IE_NAME = 'fc2'
    _NETRC_MACHINE = 'fc2'
    _TESTS = [{
        'url': 'http://video.fc2.com/en/content/20121103kUan1KHs',
        'md5': 'a6ebe8ebe0396518689d963774a54eb7',
        'info_dict': {
            'id': '20121103kUan1KHs',
            'ext': 'flv',
            'title': 'Boxing again with Puff',
        },
    }, {
        'url': 'http://video.fc2.com/en/content/20150125cEva0hDn/',
        'info_dict': {
            'id': '20150125cEva0hDn',
            'ext': 'mp4',
        },
        'params': {
            'username': 'ytdl@yt-dl.org',
            'password': '(snip)',
        },
        'skip': 'requires actual password',
    }, {
        'url': 'http://video.fc2.com/en/a/content/20130926eZpARwsF',
        'only_matching': True,
    }]

    def _login(self):
        username, password = self._get_login_info()
        if username is None or password is None:
            return False

        # Log in
        login_form_strs = {
            'email': username,
            'password': password,
            'done': 'video',
            'Submit': ' Login ',
        }

        login_data = urlencode_postdata(login_form_strs)
        request = Request(
            'https://secure.id.fc2.com/index.php?mode=login&switch_language=en', login_data)

        login_results = self._download_webpage(request, None, note='Logging in', errnote='Unable to log in')
        if 'mode=redirect&login=done' not in login_results:
            self.report_warning('unable to log in: bad username or password')
            return False

        # this is also needed
        login_redir = Request('http://id.fc2.com/?mode=redirect&login=done')
        self._download_webpage(
            login_redir, None, note='Login redirect', errnote='Login redirect failed')

        return True

    def _real_extract(self, url):
        video_id = self._match_id(url)
        self._login()
        webpage = None
        if not url.startswith('fc2:'):
            webpage = self._download_webpage(url, video_id)
            self.cookiejar.clear_session_cookies()  # must clear
            self._login()

        title, thumbnail, description = None, None, None
        if webpage is not None:
            title = self._html_search_regex(
                (r'<h2\s+class="videoCnt_title">([^<]+?)</h2>',
                 r'\s+href="[^"]+"\s*title="([^"]+?)"\s*rel="nofollow">\s*<img',
                 # there's two matches in the webpage
                 r'\s+href="[^"]+"\s*title="([^"]+?)"\s*rel="nofollow">\s*\1'),
                webpage,
                'title', fatal=False)
            thumbnail = self._og_search_thumbnail(webpage)
            description = self._og_search_description(webpage, default=None)

        vidplaylist = self._download_json(
            'https://video.fc2.com/api/v3/videoplaylist/%s?sh=1&fs=0' % video_id, video_id,
            note='Downloading info page')
        vid_url = traverse_obj(vidplaylist, ('playlist', 'nq'))
        if not vid_url:
            raise ExtractorError('Unable to extract video URL')
        vid_url = urljoin('https://video.fc2.com/', vid_url)

        return {
            'id': video_id,
            'title': title,
            'url': vid_url,
            'ext': 'mp4',
            'protocol': 'm3u8_native',
            'description': description,
            'thumbnail': thumbnail,
        }


class FC2EmbedIE(InfoExtractor):
    _VALID_URL = r'https?://video\.fc2\.com/flv2\.swf\?(?P<query>.+)'
    IE_NAME = 'fc2:embed'

    _TEST = {
        'url': 'http://video.fc2.com/flv2.swf?t=201404182936758512407645&i=20130316kwishtfitaknmcgd76kjd864hso93htfjcnaogz629mcgfs6rbfk0hsycma7shkf85937cbchfygd74&i=201403223kCqB3Ez&d=2625&sj=11&lang=ja&rel=1&from=11&cmt=1&tk=TlRBM09EQTNNekU9&tl=プリズン･ブレイク%20S1-01%20マイケル%20【吹替】',
        'md5': 'b8aae5334cb691bdb1193a88a6ab5d5a',
        'info_dict': {
            'id': '201403223kCqB3Ez',
            'ext': 'flv',
            'title': 'プリズン･ブレイク S1-01 マイケル 【吹替】',
            'thumbnail': r're:^https?://.*\.jpg$',
        },
    }

    def _real_extract(self, url):
        mobj = self._match_valid_url(url)
        query = compat_parse_qs(mobj.group('query'))

        video_id = query['i'][-1]
        title = query.get('tl', ['FC2 video %s' % video_id])[0]

        sj = query.get('sj', [None])[0]
        thumbnail = None
        if sj:
            # See thumbnailImagePath() in ServerConst.as of flv2.swf
            thumbnail = 'http://video%s-thumbnail.fc2.com/up/pic/%s.jpg' % (
                sj, '/'.join((video_id[:6], video_id[6:8], video_id[-2], video_id[-1], video_id)))

        return {
            '_type': 'url_transparent',
            'ie_key': FC2IE.ie_key(),
            'url': 'fc2:%s' % video_id,
            'title': title,
            'thumbnail': thumbnail,
        }


class FC2LiveIE(InfoExtractor):
    _VALID_URL = r'https?://live\.fc2\.com/(?P<id>\d+)'
    IE_NAME = 'fc2:live'

    _TESTS = [{
        'url': 'https://live.fc2.com/57892267/',
        'info_dict': {
            'id': '57892267',
            'title': 'どこまで・・・',
            'uploader': 'あつあげ',
            'uploader_id': '57892267',
            'thumbnail': r're:https?://.+fc2.+',
        },
        'skip': 'livestream',
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage('https://live.fc2.com/%s/' % video_id, video_id)

        self._set_cookie('live.fc2.com', 'js-player_size', '1')

        member_api = self._download_json(
            'https://live.fc2.com/api/memberApi.php', video_id, data=urlencode_postdata({
                'channel': '1',
                'profile': '1',
                'user': '1',
                'streamid': video_id
            }), note='Requesting member info')

        control_server = self._download_json(
            'https://live.fc2.com/api/getControlServer.php', video_id, note='Downloading ControlServer data',
            data=urlencode_postdata({
                'channel_id': video_id,
                'mode': 'play',
                'orz': '',
                'channel_version': member_api['data']['channel_data']['version'],
                'client_version': '2.1.0\n [1]',
                'client_type': 'pc',
                'client_app': 'browser_hls',
                'ipv6': '',
            }), headers={'X-Requested-With': 'XMLHttpRequest'})
        self._set_cookie('live.fc2.com', 'l_ortkn', control_server['orz_raw'])

        ws_url = update_url_query(control_server['url'], {'control_token': control_server['control_token']})
        playlist_data = None

        ws = self._request_webpage(Request(ws_url, headers={
            'Origin': 'https://live.fc2.com',
        }), video_id, note='Fetching HLS playlist info via WebSocket')

        self.write_debug('Sending HLS server request')

        while True:
            recv = ws.recv()
            if not recv:
                continue
            data = self._parse_json(recv, video_id, fatal=False)
            if not data or not isinstance(data, dict):
                continue

            if data.get('name') == 'connect_complete':
                break
        ws.send(r'{"name":"get_hls_information","arguments":{},"id":1}')

        while True:
            recv = ws.recv()
            if not recv:
                continue
            data = self._parse_json(recv, video_id, fatal=False)
            if not data or not isinstance(data, dict):
                continue
            if data.get('name') == '_response_' and data.get('id') == 1:
                self.write_debug('Goodbye')
                playlist_data = data
                break
            self.write_debug('Server said: %s%s' % (recv[:100], '...' if len(recv) > 100 else ''))

        if not playlist_data:
            raise ExtractorError('Unable to fetch HLS playlist info via WebSocket')

        formats = []
        for name, playlists in playlist_data['arguments'].items():
            if not isinstance(playlists, list):
                continue
            for pl in playlists:
                if pl.get('status') == 0 and 'master_playlist' in pl.get('url'):
                    formats.extend(self._extract_m3u8_formats(
                        pl['url'], video_id, ext='mp4', m3u8_id=name, live=True,
                        headers={
                            'Origin': 'https://live.fc2.com',
                            'Referer': url,
                        }))

        for fmt in formats:
            fmt.update({
                'protocol': 'fc2_live',
                'ws': ws,
            })

        title = self._html_search_meta(('og:title', 'twitter:title'), webpage, 'live title', fatal=False)
        if not title:
            title = self._html_extract_title(webpage, 'html title', fatal=False)
            if title:
                # remove service name in <title>
                title = re.sub(r'\s+-\s+.+$', '', title)
        uploader = None
        if title:
            match = self._search_regex(r'^(.+?)\s*\[(.+?)\]$', title, 'title and uploader', default=None, group=(1, 2))
            if match and all(match):
                title, uploader = match

        live_info_view = self._search_regex(r'(?s)liveInfoView\s*:\s*({.+?}),\s*premiumStateView', webpage, 'user info', fatal=False) or None
        if live_info_view:
            # remove jQuery code from object literal
            live_info_view = re.sub(r'\$\(.+?\)[^,]+,', '"",', live_info_view)
            live_info_view = self._parse_json(js_to_json(live_info_view), video_id)

        return {
            'id': video_id,
            'title': title or traverse_obj(live_info_view, 'title'),
            'description': self._html_search_meta(
                ('og:description', 'twitter:description'),
                webpage, 'live description', fatal=False) or traverse_obj(live_info_view, 'info'),
            'formats': formats,
            'uploader': uploader or traverse_obj(live_info_view, 'name'),
            'uploader_id': video_id,
            'thumbnail': traverse_obj(live_info_view, 'thumb'),
            'is_live': True,
        }
Commit	Line	Data
15dfb392 LNO	1	import re
15dfb392 LNO	2
8e71456a	3	from .common import InfoExtractor
8a82af35	4	from ..compat import compat_parse_qs
3d2623a8	5	from ..networking import Request
1cc79574 PH	6	from ..utils import (
1cc79574 PH	7	ExtractorError,
15dfb392	8	js_to_json,
d6bc443b	9	traverse_obj,
15dfb392	10	update_url_query,
6e6bc8da	11	urlencode_postdata,
d6bc443b	12	urljoin,
1cc79574	13	)
8e71456a PH	14
	15
	16	class FC2IE(InfoExtractor):
cf0efe96	17	_VALID_URL = r'^(?:https?://video\.fc2\.com/(?:[^/]+/)*content/\|fc2:)(?P<id>[^/]+)'
8e71456a	18	IE_NAME = 'fc2'
4231235c	19	_NETRC_MACHINE = 'fc2'
8940b860	20	_TESTS = [{
8e71456a PH	21	'url': 'http://video.fc2.com/en/content/20121103kUan1KHs',
	22	'md5': 'a6ebe8ebe0396518689d963774a54eb7',
	23	'info_dict': {
	24	'id': '20121103kUan1KHs',
	25	'ext': 'flv',
	26	'title': 'Boxing again with Puff',
	27	},
8940b860 PH	28	}, {
	29	'url': 'http://video.fc2.com/en/content/20150125cEva0hDn/',
	30	'info_dict': {
	31	'id': '20150125cEva0hDn',
	32	'ext': 'mp4',
	33	},
	34	'params': {
	35	'username': 'ytdl@yt-dl.org',
	36	'password': '(snip)',
38d05d17 YCH	37	},
38d05d17 YCH	38	'skip': 'requires actual password',
52dfb7ff S	39	}, {
	40	'url': 'http://video.fc2.com/en/a/content/20130926eZpARwsF',
	41	'only_matching': True,
8940b860	42	}]
4231235c	43
4231235c	44	def _login(self):
68217024	45	username, password = self._get_login_info()
8940b860 PH	46	if username is None or password is None:
8940b860 PH	47	return False
4231235c	48
	49	# Log in
	50	login_form_strs = {
8940b860	51	'email': username,
4231235c	52	'password': password,
8940b860 PH	53	'done': 'video',
8940b860 PH	54	'Submit': ' Login ',
4231235c	55	}
4231235c	56
6e6bc8da	57	login_data = urlencode_postdata(login_form_strs)
3d2623a8	58	request = Request(
4231235c	59	'https://secure.id.fc2.com/index.php?mode=login&switch_language=en', login_data)
	60
	61	login_results = self._download_webpage(request, None, note='Logging in', errnote='Unable to log in')
	62	if 'mode=redirect&login=done' not in login_results:
8940b860	63	self.report_warning('unable to log in: bad username or password')
4231235c	64	return False
8940b860	65
4231235c	66	# this is also needed
3d2623a8	67	login_redir = Request('http://id.fc2.com/?mode=redirect&login=done')
8940b860 PH	68	self._download_webpage(
8940b860 PH	69	login_redir, None, note='Login redirect', errnote='Login redirect failed')
4231235c	70
4231235c	71	return True
8e71456a PH	72
8e71456a PH	73	def _real_extract(self, url):
1cc79574	74	video_id = self._match_id(url)
8940b860	75	self._login()
cf0efe96 YCH	76	webpage = None
	77	if not url.startswith('fc2:'):
	78	webpage = self._download_webpage(url, video_id)
9809740b	79	self.cookiejar.clear_session_cookies() # must clear
cf0efe96 YCH	80	self._login()
cf0efe96 YCH	81
d6bc443b	82	title, thumbnail, description = None, None, None
cf0efe96	83	if webpage is not None:
d6bc443b LNO	84	title = self._html_search_regex(
	85	(r'<h2\s+class="videoCnt_title">([^<]+?)</h2>',
	86	r'\s+href="[^"]+"\stitle="([^"]+?)"\srel="nofollow">\s*<img',
	87	# there's two matches in the webpage
	88	r'\s+href="[^"]+"\stitle="([^"]+?)"\srel="nofollow">\s*\1'),
	89	webpage,
	90	'title', fatal=False)
cf0efe96	91	thumbnail = self._og_search_thumbnail(webpage)
81c5f44c	92	description = self._og_search_description(webpage, default=None)
8e71456a	93
d6bc443b LNO	94	vidplaylist = self._download_json(
	95	'https://video.fc2.com/api/v3/videoplaylist/%s?sh=1&fs=0' % video_id, video_id,
	96	note='Downloading info page')
	97	vid_url = traverse_obj(vidplaylist, ('playlist', 'nq'))
	98	if not vid_url:
	99	raise ExtractorError('Unable to extract video URL')
	100	vid_url = urljoin('https://video.fc2.com/', vid_url)
8e71456a PH	101
	102	return {
	103	'id': video_id,
23ae281b	104	'title': title,
d6bc443b LNO	105	'url': vid_url,
d6bc443b LNO	106	'ext': 'mp4',
81c5f44c	107	'protocol': 'm3u8_native',
d6bc443b	108	'description': description,
8e71456a PH	109	'thumbnail': thumbnail,
8e71456a PH	110	}
cf0efe96 YCH	111
	112
	113	class FC2EmbedIE(InfoExtractor):
	114	_VALID_URL = r'https?://video\.fc2\.com/flv2\.swf\?(?P<query>.+)'
	115	IE_NAME = 'fc2:embed'
	116
	117	_TEST = {
	118	'url': 'http://video.fc2.com/flv2.swf?t=201404182936758512407645&i=20130316kwishtfitaknmcgd76kjd864hso93htfjcnaogz629mcgfs6rbfk0hsycma7shkf85937cbchfygd74&i=201403223kCqB3Ez&d=2625&sj=11&lang=ja&rel=1&from=11&cmt=1&tk=TlRBM09EQTNNekU9&tl=プリズン･ブレイク%20S1-01%20マイケル%20【吹替】',
	119	'md5': 'b8aae5334cb691bdb1193a88a6ab5d5a',
	120	'info_dict': {
	121	'id': '201403223kCqB3Ez',
	122	'ext': 'flv',
	123	'title': 'プリズン･ブレイク S1-01 マイケル【吹替】',
ec85ded8	124	'thumbnail': r're:^https?://.*\.jpg$',
cf0efe96 YCH	125	},
	126	}
	127
	128	def _real_extract(self, url):
5ad28e7f	129	mobj = self._match_valid_url(url)
cf0efe96 YCH	130	query = compat_parse_qs(mobj.group('query'))
	131
	132	video_id = query['i'][-1]
	133	title = query.get('tl', ['FC2 video %s' % video_id])[0]
	134
	135	sj = query.get('sj', [None])[0]
	136	thumbnail = None
	137	if sj:
	138	# See thumbnailImagePath() in ServerConst.as of flv2.swf
	139	thumbnail = 'http://video%s-thumbnail.fc2.com/up/pic/%s.jpg' % (
	140	sj, '/'.join((video_id[:6], video_id[6:8], video_id[-2], video_id[-1], video_id)))
	141
	142	return {
	143	'_type': 'url_transparent',
ed2bfe93	144	'ie_key': FC2IE.ie_key(),
cf0efe96 YCH	145	'url': 'fc2:%s' % video_id,
	146	'title': title,
	147	'thumbnail': thumbnail,
	148	}
15dfb392 LNO	149
	150
	151	class FC2LiveIE(InfoExtractor):
	152	_VALID_URL = r'https?://live\.fc2\.com/(?P<id>\d+)'
	153	IE_NAME = 'fc2:live'
	154
	155	_TESTS = [{
	156	'url': 'https://live.fc2.com/57892267/',
	157	'info_dict': {
	158	'id': '57892267',
	159	'title': 'どこまで・・・',
	160	'uploader': 'あつあげ',
	161	'uploader_id': '57892267',
	162	'thumbnail': r're:https?://.+fc2.+',
	163	},
	164	'skip': 'livestream',
	165	}]
	166
	167	def _real_extract(self, url):
15dfb392 LNO	168	video_id = self._match_id(url)
	169	webpage = self._download_webpage('https://live.fc2.com/%s/' % video_id, video_id)
	170
	171	self._set_cookie('live.fc2.com', 'js-player_size', '1')
	172
	173	member_api = self._download_json(
	174	'https://live.fc2.com/api/memberApi.php', video_id, data=urlencode_postdata({
	175	'channel': '1',
	176	'profile': '1',
	177	'user': '1',
	178	'streamid': video_id
	179	}), note='Requesting member info')
	180
	181	control_server = self._download_json(
	182	'https://live.fc2.com/api/getControlServer.php', video_id, note='Downloading ControlServer data',
	183	data=urlencode_postdata({
	184	'channel_id': video_id,
	185	'mode': 'play',
	186	'orz': '',
	187	'channel_version': member_api['data']['channel_data']['version'],
	188	'client_version': '2.1.0\n [1]',
	189	'client_type': 'pc',
	190	'client_app': 'browser_hls',
	191	'ipv6': '',
	192	}), headers={'X-Requested-With': 'XMLHttpRequest'})
	193	self._set_cookie('live.fc2.com', 'l_ortkn', control_server['orz_raw'])
	194
	195	ws_url = update_url_query(control_server['url'], {'control_token': control_server['control_token']})
	196	playlist_data = None
	197
ccfd70f4	198	ws = self._request_webpage(Request(ws_url, headers={
15dfb392	199	'Origin': 'https://live.fc2.com',
ccfd70f4	200	}), video_id, note='Fetching HLS playlist info via WebSocket')
15dfb392	201
8a82af35	202	self.write_debug('Sending HLS server request')
15dfb392 LNO	203
	204	while True:
	205	recv = ws.recv()
	206	if not recv:
	207	continue
	208	data = self._parse_json(recv, video_id, fatal=False)
	209	if not data or not isinstance(data, dict):
	210	continue
	211
	212	if data.get('name') == 'connect_complete':
	213	break
	214	ws.send(r'{"name":"get_hls_information","arguments":{},"id":1}')
	215
	216	while True:
	217	recv = ws.recv()
	218	if not recv:
	219	continue
	220	data = self._parse_json(recv, video_id, fatal=False)
	221	if not data or not isinstance(data, dict):
	222	continue
	223	if data.get('name') == '_response_' and data.get('id') == 1:
8a82af35	224	self.write_debug('Goodbye')
15dfb392 LNO	225	playlist_data = data
15dfb392 LNO	226	break
8a82af35	227	self.write_debug('Server said: %s%s' % (recv[:100], '...' if len(recv) > 100 else ''))
15dfb392 LNO	228
	229	if not playlist_data:
	230	raise ExtractorError('Unable to fetch HLS playlist info via WebSocket')
	231
	232	formats = []
	233	for name, playlists in playlist_data['arguments'].items():
	234	if not isinstance(playlists, list):
	235	continue
	236	for pl in playlists:
	237	if pl.get('status') == 0 and 'master_playlist' in pl.get('url'):
	238	formats.extend(self._extract_m3u8_formats(
	239	pl['url'], video_id, ext='mp4', m3u8_id=name, live=True,
	240	headers={
	241	'Origin': 'https://live.fc2.com',
	242	'Referer': url,
	243	}))
	244
15dfb392 LNO	245	for fmt in formats:
	246	fmt.update({
	247	'protocol': 'fc2_live',
	248	'ws': ws,
	249	})
	250
	251	title = self._html_search_meta(('og:title', 'twitter:title'), webpage, 'live title', fatal=False)
	252	if not title:
	253	title = self._html_extract_title(webpage, 'html title', fatal=False)
	254	if title:
	255	# remove service name in <title>
	256	title = re.sub(r'\s+-\s+.+$', '', title)
	257	uploader = None
	258	if title:
	259	match = self._search_regex(r'^(.+?)\s*\[(.+?)\]$', title, 'title and uploader', default=None, group=(1, 2))
	260	if match and all(match):
	261	title, uploader = match
	262
	263	live_info_view = self._search_regex(r'(?s)liveInfoView\s:\s({.+?}),\s*premiumStateView', webpage, 'user info', fatal=False) or None
	264	if live_info_view:
	265	# remove jQuery code from object literal
	266	live_info_view = re.sub(r'\$\(.+?\)[^,]+,', '"",', live_info_view)
	267	live_info_view = self._parse_json(js_to_json(live_info_view), video_id)
	268
	269	return {
	270	'id': video_id,
	271	'title': title or traverse_obj(live_info_view, 'title'),
	272	'description': self._html_search_meta(
	273	('og:description', 'twitter:description'),
	274	webpage, 'live description', fatal=False) or traverse_obj(live_info_view, 'info'),
	275	'formats': formats,
	276	'uploader': uploader or traverse_obj(live_info_view, 'name'),
	277	'uploader_id': video_id,
	278	'thumbnail': traverse_obj(live_info_view, 'thumb'),
	279	'is_live': True,
	280	}