[yt-dlp.git] / yt_dlp / extractor / animeondemand.py

from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import compat_str
from ..utils import (
    determine_ext,
    extract_attributes,
    ExtractorError,
    url_or_none,
    urlencode_postdata,
    urljoin,
)


class AnimeOnDemandIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?anime-on-demand\.de/anime/(?P<id>\d+)'
    _LOGIN_URL = 'https://www.anime-on-demand.de/users/sign_in'
    _APPLY_HTML5_URL = 'https://www.anime-on-demand.de/html5apply'
    _NETRC_MACHINE = 'animeondemand'
    # German-speaking countries of Europe
    _GEO_COUNTRIES = ['AT', 'CH', 'DE', 'LI', 'LU']
    _TESTS = [{
        # jap, OmU
        'url': 'https://www.anime-on-demand.de/anime/161',
        'info_dict': {
            'id': '161',
            'title': 'Grimgar, Ashes and Illusions (OmU)',
            'description': 'md5:6681ce3c07c7189d255ac6ab23812d31',
        },
        'playlist_mincount': 4,
    }, {
        # Film wording is used instead of Episode, ger/jap, Dub/OmU
        'url': 'https://www.anime-on-demand.de/anime/39',
        'only_matching': True,
    }, {
        # Episodes without titles, jap, OmU
        'url': 'https://www.anime-on-demand.de/anime/162',
        'only_matching': True,
    }, {
        # ger/jap, Dub/OmU, account required
        'url': 'https://www.anime-on-demand.de/anime/169',
        'only_matching': True,
    }, {
        # Full length film, non-series, ger/jap, Dub/OmU, account required
        'url': 'https://www.anime-on-demand.de/anime/185',
        'only_matching': True,
    }, {
        # Flash videos
        'url': 'https://www.anime-on-demand.de/anime/12',
        'only_matching': True,
    }]

    def _login(self):
        username, password = self._get_login_info()
        if username is None:
            return

        login_page = self._download_webpage(
            self._LOGIN_URL, None, 'Downloading login page')

        if '>Our licensing terms allow the distribution of animes only to German-speaking countries of Europe' in login_page:
            self.raise_geo_restricted(
                '%s is only available in German-speaking countries of Europe' % self.IE_NAME)

        login_form = self._form_hidden_inputs('new_user', login_page)

        login_form.update({
            'user[login]': username,
            'user[password]': password,
        })

        post_url = self._search_regex(
            r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page,
            'post url', default=self._LOGIN_URL, group='url')

        if not post_url.startswith('http'):
            post_url = urljoin(self._LOGIN_URL, post_url)

        response = self._download_webpage(
            post_url, None, 'Logging in',
            data=urlencode_postdata(login_form), headers={
                'Referer': self._LOGIN_URL,
            })

        if all(p not in response for p in ('>Logout<', 'href="/users/sign_out"')):
            error = self._search_regex(
                r'<p[^>]+\bclass=(["\'])(?:(?!\1).)*\balert\b(?:(?!\1).)*\1[^>]*>(?P<error>.+?)</p>',
                response, 'error', default=None, group='error')
            if error:
                raise ExtractorError('Unable to login: %s' % error, expected=True)
            raise ExtractorError('Unable to log in')

    def _real_initialize(self):
        self._login()

    def _real_extract(self, url):
        anime_id = self._match_id(url)

        webpage = self._download_webpage(url, anime_id)

        if 'data-playlist=' not in webpage:
            self._download_webpage(
                self._APPLY_HTML5_URL, anime_id,
                'Activating HTML5 beta', 'Unable to apply HTML5 beta')
            webpage = self._download_webpage(url, anime_id)

        csrf_token = self._html_search_meta(
            'csrf-token', webpage, 'csrf token', fatal=True)

        anime_title = self._html_search_regex(
            r'(?s)<h1[^>]+itemprop="name"[^>]*>(.+?)</h1>',
            webpage, 'anime name')
        anime_description = self._html_search_regex(
            r'(?s)<div[^>]+itemprop="description"[^>]*>(.+?)</div>',
            webpage, 'anime description', default=None)

        def extract_info(html, video_id, num=None):
            title, description = [None] * 2
            formats = []

            for input_ in re.findall(
                    r'<input[^>]+class=["\'].*?streamstarter[^>]+>', html):
                attributes = extract_attributes(input_)
                title = attributes.get('data-dialog-header')
                playlist_urls = []
                for playlist_key in ('data-playlist', 'data-otherplaylist', 'data-stream'):
                    playlist_url = attributes.get(playlist_key)
                    if isinstance(playlist_url, compat_str) and re.match(
                            r'/?[\da-zA-Z]+', playlist_url):
                        playlist_urls.append(attributes[playlist_key])
                if not playlist_urls:
                    continue

                lang = attributes.get('data-lang')
                lang_note = attributes.get('value')

                for playlist_url in playlist_urls:
                    kind = self._search_regex(
                        r'videomaterialurl/\d+/([^/]+)/',
                        playlist_url, 'media kind', default=None)
                    format_id_list = []
                    if lang:
                        format_id_list.append(lang)
                    if kind:
                        format_id_list.append(kind)
                    if not format_id_list and num is not None:
                        format_id_list.append(compat_str(num))
                    format_id = '-'.join(format_id_list)
                    format_note = ', '.join(filter(None, (kind, lang_note)))
                    item_id_list = []
                    if format_id:
                        item_id_list.append(format_id)
                    item_id_list.append('videomaterial')
                    playlist = self._download_json(
                        urljoin(url, playlist_url), video_id,
                        'Downloading %s JSON' % ' '.join(item_id_list),
                        headers={
                            'X-Requested-With': 'XMLHttpRequest',
                            'X-CSRF-Token': csrf_token,
                            'Referer': url,
                            'Accept': 'application/json, text/javascript, */*; q=0.01',
                        }, fatal=False)
                    if not playlist:
                        continue
                    stream_url = url_or_none(playlist.get('streamurl'))
                    if stream_url:
                        rtmp = re.search(
                            r'^(?P<url>rtmpe?://(?P<host>[^/]+)/(?P<app>.+/))(?P<playpath>mp[34]:.+)',
                            stream_url)
                        if rtmp:
                            formats.append({
                                'url': rtmp.group('url'),
                                'app': rtmp.group('app'),
                                'play_path': rtmp.group('playpath'),
                                'page_url': url,
                                'player_url': 'https://www.anime-on-demand.de/assets/jwplayer.flash-55abfb34080700304d49125ce9ffb4a6.swf',
                                'rtmp_real_time': True,
                                'format_id': 'rtmp',
                                'ext': 'flv',
                            })
                            continue
                    start_video = playlist.get('startvideo', 0)
                    playlist = playlist.get('playlist')
                    if not playlist or not isinstance(playlist, list):
                        continue
                    playlist = playlist[start_video]
                    title = playlist.get('title')
                    if not title:
                        continue
                    description = playlist.get('description')
                    for source in playlist.get('sources', []):
                        file_ = source.get('file')
                        if not file_:
                            continue
                        ext = determine_ext(file_)
                        format_id_list = [lang, kind]
                        if ext == 'm3u8':
                            format_id_list.append('hls')
                        elif source.get('type') == 'video/dash' or ext == 'mpd':
                            format_id_list.append('dash')
                        format_id = '-'.join(filter(None, format_id_list))
                        if ext == 'm3u8':
                            file_formats = self._extract_m3u8_formats(
                                file_, video_id, 'mp4',
                                entry_protocol='m3u8_native', m3u8_id=format_id, fatal=False)
                        elif source.get('type') == 'video/dash' or ext == 'mpd':
                            continue
                            file_formats = self._extract_mpd_formats(
                                file_, video_id, mpd_id=format_id, fatal=False)
                        else:
                            continue
                        for f in file_formats:
                            f.update({
                                'language': lang,
                                'format_note': format_note,
                            })
                        formats.extend(file_formats)

            return {
                'title': title,
                'description': description,
                'formats': formats,
            }

        def extract_entries(html, video_id, common_info, num=None):
            info = extract_info(html, video_id, num)

            if info['formats']:
                self._sort_formats(info['formats'])
                f = common_info.copy()
                f.update(info)
                yield f

            # Extract teaser/trailer only when full episode is not available
            if not info['formats']:
                m = re.search(
                    r'data-dialog-header=(["\'])(?P<title>.+?)\1[^>]+href=(["\'])(?P<href>.+?)\3[^>]*>(?P<kind>Teaser|Trailer)<',
                    html)
                if m:
                    f = common_info.copy()
                    f.update({
                        'id': '%s-%s' % (f['id'], m.group('kind').lower()),
                        'title': m.group('title'),
                        'url': urljoin(url, m.group('href')),
                    })
                    yield f

        def extract_episodes(html):
            for num, episode_html in enumerate(re.findall(
                    r'(?s)<h3[^>]+class="episodebox-title".+?>Episodeninhalt<', html), 1):
                episodebox_title = self._search_regex(
                    (r'class="episodebox-title"[^>]+title=(["\'])(?P<title>.+?)\1',
                     r'class="episodebox-title"[^>]+>(?P<title>.+?)<'),
                    episode_html, 'episodebox title', default=None, group='title')
                if not episodebox_title:
                    continue

                episode_number = int(self._search_regex(
                    r'(?:Episode|Film)\s*(\d+)',
                    episodebox_title, 'episode number', default=num))
                episode_title = self._search_regex(
                    r'(?:Episode|Film)\s*\d+\s*-\s*(.+)',
                    episodebox_title, 'episode title', default=None)

                video_id = 'episode-%d' % episode_number

                common_info = {
                    'id': video_id,
                    'series': anime_title,
                    'episode': episode_title,
                    'episode_number': episode_number,
                }

                for e in extract_entries(episode_html, video_id, common_info):
                    yield e

        def extract_film(html, video_id):
            common_info = {
                'id': anime_id,
                'title': anime_title,
                'description': anime_description,
            }
            for e in extract_entries(html, video_id, common_info):
                yield e

        def entries():
            has_episodes = False
            for e in extract_episodes(webpage):
                has_episodes = True
                yield e

            if not has_episodes:
                for e in extract_film(webpage, anime_id):
                    yield e

        return self.playlist_result(
            entries(), anime_id, anime_title, anime_description)
Commit	Line	Data
e2bd68c9 S	1	from __future__ import unicode_literals
	2
	3	import re
	4
	5	from .common import InfoExtractor
2f483758	6	from ..compat import compat_str
e2bd68c9 S	7	from ..utils import (
e2bd68c9 S	8	determine_ext,
3c5d183c	9	extract_attributes,
e2bd68c9	10	ExtractorError,
3052a30d	11	url_or_none,
e2bd68c9	12	urlencode_postdata,
2f483758	13	urljoin,
e2bd68c9 S	14	)
	15
	16
	17	class AnimeOnDemandIE(InfoExtractor):
	18	_VALID_URL = r'https?://(?:www\.)?anime-on-demand\.de/anime/(?P<id>\d+)'
	19	_LOGIN_URL = 'https://www.anime-on-demand.de/users/sign_in'
	20	_APPLY_HTML5_URL = 'https://www.anime-on-demand.de/html5apply'
b4561e85	21	_NETRC_MACHINE = 'animeondemand'
018cc615 S	22	# German-speaking countries of Europe
018cc615 S	23	_GEO_COUNTRIES = ['AT', 'CH', 'DE', 'LI', 'LU']
b57fecfd	24	_TESTS = [{
1f7258a3	25	# jap, OmU
e2bd68c9 S	26	'url': 'https://www.anime-on-demand.de/anime/161',
	27	'info_dict': {
	28	'id': '161',
	29	'title': 'Grimgar, Ashes and Illusions (OmU)',
	30	'description': 'md5:6681ce3c07c7189d255ac6ab23812d31',
	31	},
	32	'playlist_mincount': 4,
b57fecfd	33	}, {
1f7258a3	34	# Film wording is used instead of Episode, ger/jap, Dub/OmU
b57fecfd S	35	'url': 'https://www.anime-on-demand.de/anime/39',
b57fecfd S	36	'only_matching': True,
85e8f26b	37	}, {
1f7258a3	38	# Episodes without titles, jap, OmU
85e8f26b S	39	'url': 'https://www.anime-on-demand.de/anime/162',
85e8f26b S	40	'only_matching': True,
3c5d183c S	41	}, {
	42	# ger/jap, Dub/OmU, account required
	43	'url': 'https://www.anime-on-demand.de/anime/169',
	44	'only_matching': True,
1f7258a3 S	45	}, {
	46	# Full length film, non-series, ger/jap, Dub/OmU, account required
	47	'url': 'https://www.anime-on-demand.de/anime/185',
	48	'only_matching': True,
2709d9fa S	49	}, {
	50	# Flash videos
	51	'url': 'https://www.anime-on-demand.de/anime/12',
	52	'only_matching': True,
b57fecfd	53	}]
e2bd68c9 S	54
e2bd68c9 S	55	def _login(self):
68217024	56	username, password = self._get_login_info()
e2bd68c9 S	57	if username is None:
	58	return
	59
	60	login_page = self._download_webpage(
	61	self._LOGIN_URL, None, 'Downloading login page')
	62
3e8bb9a9 S	63	if '>Our licensing terms allow the distribution of animes only to German-speaking countries of Europe' in login_page:
	64	self.raise_geo_restricted(
	65	'%s is only available in German-speaking countries of Europe' % self.IE_NAME)
	66
e2bd68c9 S	67	login_form = self._form_hidden_inputs('new_user', login_page)
	68
	69	login_form.update({
	70	'user[login]': username,
	71	'user[password]': password,
	72	})
	73
	74	post_url = self._search_regex(
	75	r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page,
	76	'post url', default=self._LOGIN_URL, group='url')
	77
	78	if not post_url.startswith('http'):
2f483758	79	post_url = urljoin(self._LOGIN_URL, post_url)
e2bd68c9 S	80
e2bd68c9 S	81	response = self._download_webpage(
e4d95865	82	post_url, None, 'Logging in',
2f483758 S	83	data=urlencode_postdata(login_form), headers={
	84	'Referer': self._LOGIN_URL,
	85	})
e2bd68c9 S	86
	87	if all(p not in response for p in ('>Logout<', 'href="/users/sign_out"')):
	88	error = self._search_regex(
d2d766bc	89	r'<p[^>]+\bclass=(["\'])(?:(?!\1).)\balert\b(?:(?!\1).)\1[^>]*>(?P<error>.+?)</p>',
17c3aced	90	response, 'error', default=None, group='error')
e2bd68c9 S	91	if error:
	92	raise ExtractorError('Unable to login: %s' % error, expected=True)
	93	raise ExtractorError('Unable to log in')
	94
	95	def _real_initialize(self):
	96	self._login()
	97
	98	def _real_extract(self, url):
	99	anime_id = self._match_id(url)
	100
	101	webpage = self._download_webpage(url, anime_id)
	102
	103	if 'data-playlist=' not in webpage:
	104	self._download_webpage(
	105	self._APPLY_HTML5_URL, anime_id,
	106	'Activating HTML5 beta', 'Unable to apply HTML5 beta')
	107	webpage = self._download_webpage(url, anime_id)
	108
	109	csrf_token = self._html_search_meta(
	110	'csrf-token', webpage, 'csrf token', fatal=True)
	111
	112	anime_title = self._html_search_regex(
	113	r'(?s)<h1[^>]+itemprop="name"[^>]*>(.+?)</h1>',
	114	webpage, 'anime name')
	115	anime_description = self._html_search_regex(
	116	r'(?s)<div[^>]+itemprop="description"[^>]*>(.+?)</div>',
	117	webpage, 'anime description', default=None)
	118
9e4f5dc1	119	def extract_info(html, video_id, num=None):
1f7258a3	120	title, description = [None] * 2
e2bd68c9 S	121	formats = []
e2bd68c9 S	122
3c5d183c	123	for input_ in re.findall(
2709d9fa	124	r'<input[^>]+class=["\'].*?streamstarter[^>]+>', html):
3c5d183c	125	attributes = extract_attributes(input_)
2709d9fa	126	title = attributes.get('data-dialog-header')
3c5d183c	127	playlist_urls = []
2709d9fa	128	for playlist_key in ('data-playlist', 'data-otherplaylist', 'data-stream'):
3c5d183c S	129	playlist_url = attributes.get(playlist_key)
	130	if isinstance(playlist_url, compat_str) and re.match(
	131	r'/?[\da-zA-Z]+', playlist_url):
	132	playlist_urls.append(attributes[playlist_key])
	133	if not playlist_urls:
	134	continue
	135
	136	lang = attributes.get('data-lang')
	137	lang_note = attributes.get('value')
	138
	139	for playlist_url in playlist_urls:
	140	kind = self._search_regex(
	141	r'videomaterialurl/\d+/([^/]+)/',
	142	playlist_url, 'media kind', default=None)
	143	format_id_list = []
	144	if lang:
	145	format_id_list.append(lang)
	146	if kind:
	147	format_id_list.append(kind)
9e4f5dc1	148	if not format_id_list and num is not None:
5c69f7a4	149	format_id_list.append(compat_str(num))
3c5d183c S	150	format_id = '-'.join(format_id_list)
3c5d183c S	151	format_note = ', '.join(filter(None, (kind, lang_note)))
2f483758 S	152	item_id_list = []
	153	if format_id:
	154	item_id_list.append(format_id)
	155	item_id_list.append('videomaterial')
	156	playlist = self._download_json(
	157	urljoin(url, playlist_url), video_id,
	158	'Downloading %s JSON' % ' '.join(item_id_list),
3c5d183c S	159	headers={
	160	'X-Requested-With': 'XMLHttpRequest',
	161	'X-CSRF-Token': csrf_token,
	162	'Referer': url,
	163	'Accept': 'application/json, text/javascript, /; q=0.01',
2f483758	164	}, fatal=False)
3c5d183c S	165	if not playlist:
3c5d183c S	166	continue
3052a30d	167	stream_url = url_or_none(playlist.get('streamurl'))
2709d9fa S	168	if stream_url:
	169	rtmp = re.search(
	170	r'^(?P<url>rtmpe?://(?P<host>[^/]+)/(?P<app>.+/))(?P<playpath>mp[34]:.+)',
	171	stream_url)
	172	if rtmp:
	173	formats.append({
	174	'url': rtmp.group('url'),
	175	'app': rtmp.group('app'),
	176	'play_path': rtmp.group('playpath'),
	177	'page_url': url,
	178	'player_url': 'https://www.anime-on-demand.de/assets/jwplayer.flash-55abfb34080700304d49125ce9ffb4a6.swf',
	179	'rtmp_real_time': True,
	180	'format_id': 'rtmp',
	181	'ext': 'flv',
	182	})
	183	continue
5c69f7a4	184	start_video = playlist.get('startvideo', 0)
3c5d183c S	185	playlist = playlist.get('playlist')
	186	if not playlist or not isinstance(playlist, list):
	187	continue
5c69f7a4	188	playlist = playlist[start_video]
3c5d183c S	189	title = playlist.get('title')
	190	if not title:
	191	continue
e2bd68c9 S	192	description = playlist.get('description')
	193	for source in playlist.get('sources', []):
	194	file_ = source.get('file')
5c69f7a4 S	195	if not file_:
	196	continue
	197	ext = determine_ext(file_)
	198	format_id_list = [lang, kind]
	199	if ext == 'm3u8':
	200	format_id_list.append('hls')
	201	elif source.get('type') == 'video/dash' or ext == 'mpd':
	202	format_id_list.append('dash')
	203	format_id = '-'.join(filter(None, format_id_list))
	204	if ext == 'm3u8':
	205	file_formats = self._extract_m3u8_formats(
e2bd68c9	206	file_, video_id, 'mp4',
5c69f7a4 S	207	entry_protocol='m3u8_native', m3u8_id=format_id, fatal=False)
5c69f7a4 S	208	elif source.get('type') == 'video/dash' or ext == 'mpd':
bc5d16b3	209	continue
5c69f7a4 S	210	file_formats = self._extract_mpd_formats(
	211	file_, video_id, mpd_id=format_id, fatal=False)
	212	else:
	213	continue
	214	for f in file_formats:
	215	f.update({
	216	'language': lang,
	217	'format_note': format_note,
	218	})
	219	formats.extend(file_formats)
e2bd68c9	220
1f7258a3 S	221	return {
	222	'title': title,
	223	'description': description,
	224	'formats': formats,
	225	}
	226
ab52bb51	227	def extract_entries(html, video_id, common_info, num=None):
9e4f5dc1	228	info = extract_info(html, video_id, num)
1f7258a3 S	229
	230	if info['formats']:
	231	self._sort_formats(info['formats'])
e2bd68c9	232	f = common_info.copy()
1f7258a3	233	f.update(info)
30a074c2	234	yield f
e2bd68c9	235
1f7258a3 S	236	# Extract teaser/trailer only when full episode is not available
1f7258a3 S	237	if not info['formats']:
85c637b7	238	m = re.search(
1f7258a3 S	239	r'data-dialog-header=(["\'])(?P<title>.+?)\1[^>]+href=(["\'])(?P<href>.+?)\3[^>]*>(?P<kind>Teaser\|Trailer)<',
1f7258a3 S	240	html)
85c637b7 S	241	if m:
	242	f = common_info.copy()
	243	f.update({
1f7258a3	244	'id': '%s-%s' % (f['id'], m.group('kind').lower()),
85c637b7	245	'title': m.group('title'),
2f483758	246	'url': urljoin(url, m.group('href')),
85c637b7	247	})
30a074c2	248	yield f
e2bd68c9	249
1f7258a3 S	250	def extract_episodes(html):
	251	for num, episode_html in enumerate(re.findall(
	252	r'(?s)<h3[^>]+class="episodebox-title".+?>Episodeninhalt<', html), 1):
	253	episodebox_title = self._search_regex(
	254	(r'class="episodebox-title"[^>]+title=(["\'])(?P<title>.+?)\1',
	255	r'class="episodebox-title"[^>]+>(?P<title>.+?)<'),
	256	episode_html, 'episodebox title', default=None, group='title')
	257	if not episodebox_title:
	258	continue
	259
	260	episode_number = int(self._search_regex(
	261	r'(?:Episode\|Film)\s*(\d+)',
	262	episodebox_title, 'episode number', default=num))
	263	episode_title = self._search_regex(
	264	r'(?:Episode\|Film)\s\d+\s-\s*(.+)',
	265	episodebox_title, 'episode title', default=None)
	266
	267	video_id = 'episode-%d' % episode_number
	268
	269	common_info = {
	270	'id': video_id,
	271	'series': anime_title,
	272	'episode': episode_title,
	273	'episode_number': episode_number,
	274	}
	275
30a074c2	276	for e in extract_entries(episode_html, video_id, common_info):
30a074c2	277	yield e
1f7258a3 S	278
	279	def extract_film(html, video_id):
	280	common_info = {
	281	'id': anime_id,
	282	'title': anime_title,
	283	'description': anime_description,
	284	}
30a074c2	285	for e in extract_entries(html, video_id, common_info):
30a074c2	286	yield e
1f7258a3	287
30a074c2	288	def entries():
	289	has_episodes = False
	290	for e in extract_episodes(webpage):
	291	has_episodes = True
	292	yield e
1f7258a3	293
30a074c2	294	if not has_episodes:
	295	for e in extract_film(webpage, anime_id):
	296	yield e
1f7258a3	297
30a074c2	298	return self.playlist_result(
30a074c2	299	entries(), anime_id, anime_title, anime_description)