[yt-dlp.git] / yt_dlp / extractor / vvvvid.py

import re

from .common import InfoExtractor
from .youtube import YoutubeIE
from ..utils import (
    ExtractorError,
    int_or_none,
    str_or_none,
)


class VVVVIDIE(InfoExtractor):
    _VALID_URL_BASE = r'https?://(?:www\.)?vvvvid\.it/(?:#!)?(?:show|anime|film|series)/'
    _VALID_URL = r'%s(?P<show_id>\d+)/[^/]+/(?P<season_id>\d+)/(?P<id>[0-9]+)' % _VALID_URL_BASE
    _TESTS = [{
        # video_type == 'video/vvvvid'
        'url': 'https://www.vvvvid.it/#!show/434/perche-dovrei-guardarlo-di-dario-moccia/437/489048/ping-pong',
        'md5': 'b8d3cecc2e981adc3835adf07f6df91b',
        'info_dict': {
            'id': '489048',
            'ext': 'mp4',
            'title': 'Ping Pong',
            'duration': 239,
            'series': '"Perché dovrei guardarlo?" di Dario Moccia',
            'season_id': '437',
            'episode': 'Ping Pong',
            'episode_number': 1,
            'episode_id': '3334',
            'view_count': int,
            'like_count': int,
            'repost_count': int,
        },
        'params': {
            'skip_download': True,
        },
    }, {
        # video_type == 'video/rcs'
        'url': 'https://www.vvvvid.it/#!show/376/death-note-live-action/377/482493/episodio-01',
        'md5': '33e0edfba720ad73a8782157fdebc648',
        'info_dict': {
            'id': '482493',
            'ext': 'mp4',
            'title': 'Episodio 01',
        },
        'params': {
            'skip_download': True,
        },
    }, {
        # video_type == 'video/youtube'
        'url': 'https://www.vvvvid.it/show/404/one-punch-man/406/486683/trailer',
        'md5': '33e0edfba720ad73a8782157fdebc648',
        'info_dict': {
            'id': 'RzmFKUDOUgw',
            'ext': 'mp4',
            'title': 'Trailer',
            'upload_date': '20150906',
            'description': 'md5:a5e802558d35247fee285875328c0b80',
            'uploader_id': 'BandaiVisual',
            'uploader': 'BANDAI NAMCO Arts Channel',
        },
        'params': {
            'skip_download': True,
        },
    }, {
        # video_type == 'video/dash'
        'url': 'https://www.vvvvid.it/show/683/made-in-abyss/1542/693786/nanachi',
        'info_dict': {
            'id': '693786',
            'ext': 'mp4',
            'title': 'Nanachi',
        },
        'params': {
            'skip_download': True,
            'format': 'mp4',
        },
    }, {
        'url': 'https://www.vvvvid.it/show/434/perche-dovrei-guardarlo-di-dario-moccia/437/489048',
        'only_matching': True
    }]
    _conn_id = None

    def _real_initialize(self):
        self._conn_id = self._download_json(
            'https://www.vvvvid.it/user/login',
            None, headers=self.geo_verification_headers())['data']['conn_id']

    def _download_info(self, show_id, path, video_id, fatal=True, query=None):
        q = {
            'conn_id': self._conn_id,
        }
        if query:
            q.update(query)
        response = self._download_json(
            'https://www.vvvvid.it/vvvvid/ondemand/%s/%s' % (show_id, path),
            video_id, headers=self.geo_verification_headers(), query=q, fatal=fatal)
        if not (response or fatal):
            return
        if response.get('result') == 'error':
            raise ExtractorError('%s said: %s' % (
                self.IE_NAME, response['message']), expected=True)
        return response['data']

    def _extract_common_video_info(self, video_data):
        return {
            'thumbnail': video_data.get('thumbnail'),
            'episode_id': str_or_none(video_data.get('id')),
        }

    def _real_extract(self, url):
        show_id, season_id, video_id = self._match_valid_url(url).groups()

        response = self._download_info(
            show_id, 'season/%s' % season_id,
            video_id, query={'video_id': video_id})

        vid = int(video_id)
        video_data = list(filter(
            lambda episode: episode.get('video_id') == vid, response))[0]
        title = video_data['title']
        formats = []

        # vvvvid embed_info decryption algorithm is reverse engineered from function $ds(h) at vvvvid.js
        def ds(h):
            g = "MNOPIJKL89+/4567UVWXQRSTEFGHABCDcdefYZabstuvopqr0123wxyzklmnghij"

            def f(m):
                l = []
                o = 0
                b = False
                m_len = len(m)
                while ((not b) and o < m_len):
                    n = m[o] << 2
                    o += 1
                    k = -1
                    j = -1
                    if o < m_len:
                        n += m[o] >> 4
                        o += 1
                        if o < m_len:
                            k = (m[o - 1] << 4) & 255
                            k += m[o] >> 2
                            o += 1
                            if o < m_len:
                                j = (m[o - 1] << 6) & 255
                                j += m[o]
                                o += 1
                            else:
                                b = True
                        else:
                            b = True
                    else:
                        b = True
                    l.append(n)
                    if k != -1:
                        l.append(k)
                    if j != -1:
                        l.append(j)
                return l

            c = []
            for e in h:
                c.append(g.index(e))

            c_len = len(c)
            for e in range(c_len * 2 - 1, -1, -1):
                a = c[e % c_len] ^ c[(e + 1) % c_len]
                c[e % c_len] = a

            c = f(c)
            d = ''
            for e in c:
                d += chr(e)

            return d

        info = {}

        def metadata_from_url(r_url):
            if not info and r_url:
                mobj = re.search(r'_(?:S(\d+))?Ep(\d+)', r_url)
                if mobj:
                    info['episode_number'] = int(mobj.group(2))
                    season_number = mobj.group(1)
                    if season_number:
                        info['season_number'] = int(season_number)

        video_type = video_data.get('video_type')
        is_youtube = False
        for quality in ('', '_sd'):
            embed_code = video_data.get('embed_info' + quality)
            if not embed_code:
                continue
            embed_code = ds(embed_code)
            if video_type == 'video/kenc':
                embed_code = re.sub(r'https?(://[^/]+)/z/', r'https\1/i/', embed_code).replace('/manifest.f4m', '/master.m3u8')
                kenc = self._download_json(
                    'https://www.vvvvid.it/kenc', video_id, query={
                        'action': 'kt',
                        'conn_id': self._conn_id,
                        'url': embed_code,
                    }, fatal=False) or {}
                kenc_message = kenc.get('message')
                if kenc_message:
                    embed_code += '?' + ds(kenc_message)
                formats.extend(self._extract_m3u8_formats(
                    embed_code, video_id, 'mp4', m3u8_id='hls', fatal=False))
            elif video_type == 'video/rcs':
                formats.extend(self._extract_akamai_formats(embed_code, video_id))
            elif video_type == 'video/youtube':
                info.update({
                    '_type': 'url_transparent',
                    'ie_key': YoutubeIE.ie_key(),
                    'url': embed_code,
                })
                is_youtube = True
                break
            elif video_type == 'video/dash':
                formats.extend(self._extract_m3u8_formats(
                    embed_code, video_id, 'mp4', m3u8_id='hls', fatal=False))
            else:
                formats.extend(self._extract_wowza_formats(
                    'http://sb.top-ix.org/videomg/_definst_/mp4:%s/playlist.m3u8' % embed_code, video_id))
            metadata_from_url(embed_code)

        if not is_youtube:
            info['formats'] = formats

        metadata_from_url(video_data.get('thumbnail'))
        info.update(self._extract_common_video_info(video_data))
        info.update({
            'id': video_id,
            'title': title,
            'duration': int_or_none(video_data.get('length')),
            'series': video_data.get('show_title'),
            'season_id': season_id,
            'episode': title,
            'view_count': int_or_none(video_data.get('views')),
            'like_count': int_or_none(video_data.get('video_likes')),
            'repost_count': int_or_none(video_data.get('video_shares')),
        })
        return info


class VVVVIDShowIE(VVVVIDIE):  # XXX: Do not subclass from concrete IE
    _VALID_URL = r'(?P<base_url>%s(?P<id>\d+)(?:/(?P<show_title>[^/?&#]+))?)/?(?:[?#&]|$)' % VVVVIDIE._VALID_URL_BASE
    _TESTS = [{
        'url': 'https://www.vvvvid.it/show/156/psyco-pass',
        'info_dict': {
            'id': '156',
            'title': 'Psycho-Pass',
            'description': 'md5:94d572c0bd85894b193b8aebc9a3a806',
        },
        'playlist_count': 46,
    }, {
        'url': 'https://www.vvvvid.it/show/156',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        base_url, show_id, show_title = self._match_valid_url(url).groups()

        seasons = self._download_info(
            show_id, 'seasons/', show_title)

        show_info = self._download_info(
            show_id, 'info/', show_title, fatal=False)

        if not show_title:
            base_url += "/title"

        entries = []
        for season in (seasons or []):
            episodes = season.get('episodes') or []
            playlist_title = season.get('name') or show_info.get('title')
            for episode in episodes:
                if episode.get('playable') is False:
                    continue
                season_id = str_or_none(episode.get('season_id'))
                video_id = str_or_none(episode.get('video_id'))
                if not (season_id and video_id):
                    continue
                info = self._extract_common_video_info(episode)
                info.update({
                    '_type': 'url_transparent',
                    'ie_key': VVVVIDIE.ie_key(),
                    'url': '/'.join([base_url, season_id, video_id]),
                    'title': episode.get('title'),
                    'description': episode.get('description'),
                    'season_id': season_id,
                    'playlist_title': playlist_title,
                })
                entries.append(info)

        return self.playlist_result(
            entries, show_id, show_info.get('title'), show_info.get('description'))
Commit	Line	Data
7b1e8079 RA	1	import re
	2
	3	from .common import InfoExtractor
2181983a	4	from .youtube import YoutubeIE
7b1e8079 RA	5	from ..utils import (
	6	ExtractorError,
	7	int_or_none,
	8	str_or_none,
	9	)
	10
	11
	12	class VVVVIDIE(InfoExtractor):
29f7c58a	13	_VALID_URL_BASE = r'https?://(?:www\.)?vvvvid\.it/(?:#!)?(?:show\|anime\|film\|series)/'
29f7c58a	14	_VALID_URL = r'%s(?P<show_id>\d+)/[^/]+/(?P<season_id>\d+)/(?P<id>[0-9]+)' % _VALID_URL_BASE
7b1e8079 RA	15	_TESTS = [{
	16	# video_type == 'video/vvvvid'
	17	'url': 'https://www.vvvvid.it/#!show/434/perche-dovrei-guardarlo-di-dario-moccia/437/489048/ping-pong',
	18	'md5': 'b8d3cecc2e981adc3835adf07f6df91b',
	19	'info_dict': {
	20	'id': '489048',
	21	'ext': 'mp4',
	22	'title': 'Ping Pong',
29f7c58a	23	'duration': 239,
	24	'series': '"Perché dovrei guardarlo?" di Dario Moccia',
	25	'season_id': '437',
	26	'episode': 'Ping Pong',
	27	'episode_number': 1,
	28	'episode_id': '3334',
	29	'view_count': int,
	30	'like_count': int,
	31	'repost_count': int,
7b1e8079	32	},
fb61b57d RA	33	'params': {
	34	'skip_download': True,
	35	},
7b1e8079 RA	36	}, {
	37	# video_type == 'video/rcs'
	38	'url': 'https://www.vvvvid.it/#!show/376/death-note-live-action/377/482493/episodio-01',
	39	'md5': '33e0edfba720ad73a8782157fdebc648',
	40	'info_dict': {
	41	'id': '482493',
	42	'ext': 'mp4',
	43	'title': 'Episodio 01',
	44	},
fb61b57d RA	45	'params': {
	46	'skip_download': True,
	47	},
2181983a	48	}, {
	49	# video_type == 'video/youtube'
	50	'url': 'https://www.vvvvid.it/show/404/one-punch-man/406/486683/trailer',
	51	'md5': '33e0edfba720ad73a8782157fdebc648',
	52	'info_dict': {
	53	'id': 'RzmFKUDOUgw',
	54	'ext': 'mp4',
	55	'title': 'Trailer',
	56	'upload_date': '20150906',
	57	'description': 'md5:a5e802558d35247fee285875328c0b80',
	58	'uploader_id': 'BandaiVisual',
	59	'uploader': 'BANDAI NAMCO Arts Channel',
	60	},
	61	'params': {
	62	'skip_download': True,
	63	},
16d4535a	64	}, {
	65	# video_type == 'video/dash'
	66	'url': 'https://www.vvvvid.it/show/683/made-in-abyss/1542/693786/nanachi',
	67	'info_dict': {
	68	'id': '693786',
	69	'ext': 'mp4',
	70	'title': 'Nanachi',
	71	},
	72	'params': {
	73	'skip_download': True,
	74	'format': 'mp4',
	75	},
29f7c58a	76	}, {
	77	'url': 'https://www.vvvvid.it/show/434/perche-dovrei-guardarlo-di-dario-moccia/437/489048',
	78	'only_matching': True
7b1e8079 RA	79	}]
	80	_conn_id = None
	81
	82	def _real_initialize(self):
dc1f3a9f RA	83	self._conn_id = self._download_json(
	84	'https://www.vvvvid.it/user/login',
	85	None, headers=self.geo_verification_headers())['data']['conn_id']
7b1e8079	86
ec5e77c5	87	def _download_info(self, show_id, path, video_id, fatal=True, query=None):
	88	q = {
	89	'conn_id': self._conn_id,
	90	}
	91	if query:
	92	q.update(query)
7b1e8079	93	response = self._download_json(
29f7c58a	94	'https://www.vvvvid.it/vvvvid/ondemand/%s/%s' % (show_id, path),
ec5e77c5	95	video_id, headers=self.geo_verification_headers(), query=q, fatal=fatal)
29f7c58a	96	if not (response or fatal):
	97	return
	98	if response.get('result') == 'error':
7b1e8079 RA	99	raise ExtractorError('%s said: %s' % (
7b1e8079 RA	100	self.IE_NAME, response['message']), expected=True)
29f7c58a	101	return response['data']
	102
	103	def _extract_common_video_info(self, video_data):
	104	return {
	105	'thumbnail': video_data.get('thumbnail'),
	106	'episode_id': str_or_none(video_data.get('id')),
	107	}
	108
	109	def _real_extract(self, url):
5ad28e7f	110	show_id, season_id, video_id = self._match_valid_url(url).groups()
29f7c58a	111
29f7c58a	112	response = self._download_info(
ec5e77c5	113	show_id, 'season/%s' % season_id,
ec5e77c5	114	video_id, query={'video_id': video_id})
7b1e8079 RA	115
	116	vid = int(video_id)
	117	video_data = list(filter(
29f7c58a	118	lambda episode: episode.get('video_id') == vid, response))[0]
29f7c58a	119	title = video_data['title']
7b1e8079 RA	120	formats = []
	121
	122	# vvvvid embed_info decryption algorithm is reverse engineered from function $ds(h) at vvvvid.js
	123	def ds(h):
	124	g = "MNOPIJKL89+/4567UVWXQRSTEFGHABCDcdefYZabstuvopqr0123wxyzklmnghij"
	125
	126	def f(m):
	127	l = []
	128	o = 0
	129	b = False
	130	m_len = len(m)
	131	while ((not b) and o < m_len):
	132	n = m[o] << 2
	133	o += 1
	134	k = -1
	135	j = -1
	136	if o < m_len:
	137	n += m[o] >> 4
	138	o += 1
	139	if o < m_len:
	140	k = (m[o - 1] << 4) & 255
	141	k += m[o] >> 2
	142	o += 1
	143	if o < m_len:
	144	j = (m[o - 1] << 6) & 255
	145	j += m[o]
	146	o += 1
	147	else:
	148	b = True
	149	else:
	150	b = True
	151	else:
	152	b = True
	153	l.append(n)
	154	if k != -1:
	155	l.append(k)
	156	if j != -1:
	157	l.append(j)
	158	return l
	159
	160	c = []
	161	for e in h:
	162	c.append(g.index(e))
	163
	164	c_len = len(c)
	165	for e in range(c_len * 2 - 1, -1, -1):
	166	a = c[e % c_len] ^ c[(e + 1) % c_len]
	167	c[e % c_len] = a
	168
	169	c = f(c)
	170	d = ''
	171	for e in c:
	172	d += chr(e)
	173
	174	return d
	175
29f7c58a	176	info = {}
	177
	178	def metadata_from_url(r_url):
	179	if not info and r_url:
	180	mobj = re.search(r'_(?:S(\d+))?Ep(\d+)', r_url)
	181	if mobj:
	182	info['episode_number'] = int(mobj.group(2))
	183	season_number = mobj.group(1)
	184	if season_number:
	185	info['season_number'] = int(season_number)
	186
2181983a	187	video_type = video_data.get('video_type')
	188	is_youtube = False
	189	for quality in ('', '_sd'):
7b1e8079 RA	190	embed_code = video_data.get('embed_info' + quality)
	191	if not embed_code:
	192	continue
	193	embed_code = ds(embed_code)
421a4595	194	if video_type == 'video/kenc':
	195	embed_code = re.sub(r'https?(://[^/]+)/z/', r'https\1/i/', embed_code).replace('/manifest.f4m', '/master.m3u8')
	196	kenc = self._download_json(
	197	'https://www.vvvvid.it/kenc', video_id, query={
	198	'action': 'kt',
	199	'conn_id': self._conn_id,
	200	'url': embed_code,
	201	}, fatal=False) or {}
	202	kenc_message = kenc.get('message')
	203	if kenc_message:
	204	embed_code += '?' + ds(kenc_message)
	205	formats.extend(self._extract_m3u8_formats(
	206	embed_code, video_id, 'mp4', m3u8_id='hls', fatal=False))
	207	elif video_type == 'video/rcs':
29f7c58a	208	formats.extend(self._extract_akamai_formats(embed_code, video_id))
2181983a	209	elif video_type == 'video/youtube':
	210	info.update({
	211	'_type': 'url_transparent',
	212	'ie_key': YoutubeIE.ie_key(),
	213	'url': embed_code,
	214	})
	215	is_youtube = True
	216	break
16d4535a	217	elif video_type == 'video/dash':
	218	formats.extend(self._extract_m3u8_formats(
	219	embed_code, video_id, 'mp4', m3u8_id='hls', fatal=False))
7b1e8079 RA	220	else:
	221	formats.extend(self._extract_wowza_formats(
	222	'http://sb.top-ix.org/videomg/_definst_/mp4:%s/playlist.m3u8' % embed_code, video_id))
29f7c58a	223	metadata_from_url(embed_code)
29f7c58a	224
2181983a	225	if not is_youtube:
2181983a	226	info['formats'] = formats
7b1e8079	227
29f7c58a	228	metadata_from_url(video_data.get('thumbnail'))
	229	info.update(self._extract_common_video_info(video_data))
	230	info.update({
7b1e8079	231	'id': video_id,
29f7c58a	232	'title': title,
7b1e8079 RA	233	'duration': int_or_none(video_data.get('length')),
	234	'series': video_data.get('show_title'),
	235	'season_id': season_id,
29f7c58a	236	'episode': title,
7b1e8079 RA	237	'view_count': int_or_none(video_data.get('views')),
7b1e8079 RA	238	'like_count': int_or_none(video_data.get('video_likes')),
29f7c58a	239	'repost_count': int_or_none(video_data.get('video_shares')),
	240	})
	241	return info
	242
	243
6368e2e6	244	class VVVVIDShowIE(VVVVIDIE): # XXX: Do not subclass from concrete IE
29f7c58a	245	_VALID_URL = r'(?P<base_url>%s(?P<id>\d+)(?:/(?P<show_title>[^/?&#]+))?)/?(?:[?#&]\|$)' % VVVVIDIE._VALID_URL_BASE
	246	_TESTS = [{
	247	'url': 'https://www.vvvvid.it/show/156/psyco-pass',
	248	'info_dict': {
	249	'id': '156',
	250	'title': 'Psycho-Pass',
	251	'description': 'md5:94d572c0bd85894b193b8aebc9a3a806',
	252	},
	253	'playlist_count': 46,
	254	}, {
	255	'url': 'https://www.vvvvid.it/show/156',
	256	'only_matching': True,
	257	}]
	258
	259	def _real_extract(self, url):
5ad28e7f	260	base_url, show_id, show_title = self._match_valid_url(url).groups()
29f7c58a	261
	262	seasons = self._download_info(
	263	show_id, 'seasons/', show_title)
	264
	265	show_info = self._download_info(
	266	show_id, 'info/', show_title, fatal=False)
	267
ec5e77c5	268	if not show_title:
	269	base_url += "/title"
	270
29f7c58a	271	entries = []
	272	for season in (seasons or []):
	273	episodes = season.get('episodes') or []
ec5e77c5	274	playlist_title = season.get('name') or show_info.get('title')
29f7c58a	275	for episode in episodes:
	276	if episode.get('playable') is False:
	277	continue
	278	season_id = str_or_none(episode.get('season_id'))
	279	video_id = str_or_none(episode.get('video_id'))
	280	if not (season_id and video_id):
	281	continue
	282	info = self._extract_common_video_info(episode)
	283	info.update({
ec5e77c5	284	'_type': 'url_transparent',
29f7c58a	285	'ie_key': VVVVIDIE.ie_key(),
	286	'url': '/'.join([base_url, season_id, video_id]),
	287	'title': episode.get('title'),
	288	'description': episode.get('description'),
	289	'season_id': season_id,
ec5e77c5	290	'playlist_title': playlist_title,
29f7c58a	291	})
	292	entries.append(info)
	293
	294	return self.playlist_result(
	295	entries, show_id, show_info.get('title'), show_info.get('description'))