[yt-dlp.git] / youtube_dl / extractor / animeondemand.py

from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import (
    compat_urlparse,
    compat_str,
)
from ..utils import (
    determine_ext,
    encode_dict,
    extract_attributes,
    ExtractorError,
    sanitized_Request,
    urlencode_postdata,
)


class AnimeOnDemandIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?anime-on-demand\.de/anime/(?P<id>\d+)'
    _LOGIN_URL = 'https://www.anime-on-demand.de/users/sign_in'
    _APPLY_HTML5_URL = 'https://www.anime-on-demand.de/html5apply'
    _NETRC_MACHINE = 'animeondemand'
    _TESTS = [{
        'url': 'https://www.anime-on-demand.de/anime/161',
        'info_dict': {
            'id': '161',
            'title': 'Grimgar, Ashes and Illusions (OmU)',
            'description': 'md5:6681ce3c07c7189d255ac6ab23812d31',
        },
        'playlist_mincount': 4,
    }, {
        # Film wording is used instead of Episode
        'url': 'https://www.anime-on-demand.de/anime/39',
        'only_matching': True,
    }, {
        # Episodes without titles
        'url': 'https://www.anime-on-demand.de/anime/162',
        'only_matching': True,
    }, {
        # ger/jap, Dub/OmU, account required
        'url': 'https://www.anime-on-demand.de/anime/169',
        'only_matching': True,
    }]

    def _login(self):
        (username, password) = self._get_login_info()
        if username is None:
            return

        login_page = self._download_webpage(
            self._LOGIN_URL, None, 'Downloading login page')

        if '>Our licensing terms allow the distribution of animes only to German-speaking countries of Europe' in login_page:
            self.raise_geo_restricted(
                '%s is only available in German-speaking countries of Europe' % self.IE_NAME)

        login_form = self._form_hidden_inputs('new_user', login_page)

        login_form.update({
            'user[login]': username,
            'user[password]': password,
        })

        post_url = self._search_regex(
            r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page,
            'post url', default=self._LOGIN_URL, group='url')

        if not post_url.startswith('http'):
            post_url = compat_urlparse.urljoin(self._LOGIN_URL, post_url)

        request = sanitized_Request(
            post_url, urlencode_postdata(encode_dict(login_form)))
        request.add_header('Referer', self._LOGIN_URL)

        response = self._download_webpage(
            request, None, 'Logging in as %s' % username)

        if all(p not in response for p in ('>Logout<', 'href="/users/sign_out"')):
            error = self._search_regex(
                r'<p class="alert alert-danger">(.+?)</p>',
                response, 'error', default=None)
            if error:
                raise ExtractorError('Unable to login: %s' % error, expected=True)
            raise ExtractorError('Unable to log in')

    def _real_initialize(self):
        self._login()

    def _real_extract(self, url):
        anime_id = self._match_id(url)

        webpage = self._download_webpage(url, anime_id)

        if 'data-playlist=' not in webpage:
            self._download_webpage(
                self._APPLY_HTML5_URL, anime_id,
                'Activating HTML5 beta', 'Unable to apply HTML5 beta')
            webpage = self._download_webpage(url, anime_id)

        csrf_token = self._html_search_meta(
            'csrf-token', webpage, 'csrf token', fatal=True)

        anime_title = self._html_search_regex(
            r'(?s)<h1[^>]+itemprop="name"[^>]*>(.+?)</h1>',
            webpage, 'anime name')
        anime_description = self._html_search_regex(
            r'(?s)<div[^>]+itemprop="description"[^>]*>(.+?)</div>',
            webpage, 'anime description', default=None)

        entries = []

        for num, episode_html in enumerate(re.findall(
                r'(?s)<h3[^>]+class="episodebox-title".+?>Episodeninhalt<', webpage), 1):
            episodebox_title = self._search_regex(
                (r'class="episodebox-title"[^>]+title=(["\'])(?P<title>.+?)\1',
                 r'class="episodebox-title"[^>]+>(?P<title>.+?)<'),
                episode_html, 'episodebox title', default=None, group='title')
            if not episodebox_title:
                continue

            episode_number = int(self._search_regex(
                r'(?:Episode|Film)\s*(\d+)',
                episodebox_title, 'episode number', default=num))
            episode_title = self._search_regex(
                r'(?:Episode|Film)\s*\d+\s*-\s*(.+)',
                episodebox_title, 'episode title', default=None)

            video_id = 'episode-%d' % episode_number

            common_info = {
                'id': video_id,
                'series': anime_title,
                'episode': episode_title,
                'episode_number': episode_number,
            }

            formats = []

            for input_ in re.findall(
                    r'<input[^>]+class=["\'].*?streamstarter_html5[^>]+>', episode_html):
                attributes = extract_attributes(input_)
                playlist_urls = []
                for playlist_key in ('data-playlist', 'data-otherplaylist'):
                    playlist_url = attributes.get(playlist_key)
                    if isinstance(playlist_url, compat_str) and re.match(
                            r'/?[\da-zA-Z]+', playlist_url):
                        playlist_urls.append(attributes[playlist_key])
                if not playlist_urls:
                    continue

                lang = attributes.get('data-lang')
                lang_note = attributes.get('value')

                for playlist_url in playlist_urls:
                    kind = self._search_regex(
                        r'videomaterialurl/\d+/([^/]+)/',
                        playlist_url, 'media kind', default=None)
                    format_id_list = []
                    if lang:
                        format_id_list.append(lang)
                    if kind:
                        format_id_list.append(kind)
                    if not format_id_list:
                        format_id_list.append('hls')
                    format_id = '-'.join(format_id_list)
                    format_note = ', '.join(filter(None, (kind, lang_note)))
                    request = sanitized_Request(
                        compat_urlparse.urljoin(url, playlist_url),
                        headers={
                            'X-Requested-With': 'XMLHttpRequest',
                            'X-CSRF-Token': csrf_token,
                            'Referer': url,
                            'Accept': 'application/json, text/javascript, */*; q=0.01',
                        })
                    playlist = self._download_json(
                        request, video_id, 'Downloading %s playlist JSON' % format_id,
                        fatal=False)
                    if not playlist:
                        continue
                    playlist = playlist.get('playlist')
                    if not playlist or not isinstance(playlist, list):
                        continue
                    playlist = playlist[0]
                    title = playlist.get('title')
                    if not title:
                        continue
                    description = playlist.get('description')
                    for source in playlist.get('sources', []):
                        file_ = source.get('file')
                        if file_ and determine_ext(file_) == 'm3u8':
                            m3u8_formats = self._extract_m3u8_formats(
                                file_, video_id, 'mp4',
                                entry_protocol='m3u8_native', m3u8_id=format_id)
                            for f in m3u8_formats:
                                f.update({
                                    'language': lang,
                                    'format_note': format_note,
                                })
                            formats.extend(m3u8_formats)

            if formats:
                self._sort_formats(formats)
                f = common_info.copy()
                f.update({
                    'title': title,
                    'description': description,
                    'formats': formats,
                })
                entries.append(f)

            m = re.search(
                r'data-dialog-header=(["\'])(?P<title>.+?)\1[^>]+href=(["\'])(?P<href>.+?)\3[^>]*>Teaser<',
                episode_html)
            if m:
                f = common_info.copy()
                f.update({
                    'id': '%s-teaser' % f['id'],
                    'title': m.group('title'),
                    'url': compat_urlparse.urljoin(url, m.group('href')),
                })
                entries.append(f)

        return self.playlist_result(entries, anime_id, anime_title, anime_description)
Commit	Line	Data
e2bd68c9 S	1	from __future__ import unicode_literals
	2
	3	import re
	4
	5	from .common import InfoExtractor
3c5d183c S	6	from ..compat import (
	7	compat_urlparse,
	8	compat_str,
	9	)
e2bd68c9 S	10	from ..utils import (
	11	determine_ext,
	12	encode_dict,
3c5d183c	13	extract_attributes,
e2bd68c9 S	14	ExtractorError,
	15	sanitized_Request,
	16	urlencode_postdata,
	17	)
	18
	19
	20	class AnimeOnDemandIE(InfoExtractor):
	21	_VALID_URL = r'https?://(?:www\.)?anime-on-demand\.de/anime/(?P<id>\d+)'
	22	_LOGIN_URL = 'https://www.anime-on-demand.de/users/sign_in'
	23	_APPLY_HTML5_URL = 'https://www.anime-on-demand.de/html5apply'
b4561e85	24	_NETRC_MACHINE = 'animeondemand'
b57fecfd	25	_TESTS = [{
e2bd68c9 S	26	'url': 'https://www.anime-on-demand.de/anime/161',
	27	'info_dict': {
	28	'id': '161',
	29	'title': 'Grimgar, Ashes and Illusions (OmU)',
	30	'description': 'md5:6681ce3c07c7189d255ac6ab23812d31',
	31	},
	32	'playlist_mincount': 4,
b57fecfd S	33	}, {
	34	# Film wording is used instead of Episode
	35	'url': 'https://www.anime-on-demand.de/anime/39',
	36	'only_matching': True,
85e8f26b S	37	}, {
	38	# Episodes without titles
	39	'url': 'https://www.anime-on-demand.de/anime/162',
	40	'only_matching': True,
3c5d183c S	41	}, {
	42	# ger/jap, Dub/OmU, account required
	43	'url': 'https://www.anime-on-demand.de/anime/169',
	44	'only_matching': True,
b57fecfd	45	}]
e2bd68c9 S	46
	47	def _login(self):
	48	(username, password) = self._get_login_info()
	49	if username is None:
	50	return
	51
	52	login_page = self._download_webpage(
	53	self._LOGIN_URL, None, 'Downloading login page')
	54
3e8bb9a9 S	55	if '>Our licensing terms allow the distribution of animes only to German-speaking countries of Europe' in login_page:
	56	self.raise_geo_restricted(
	57	'%s is only available in German-speaking countries of Europe' % self.IE_NAME)
	58
e2bd68c9 S	59	login_form = self._form_hidden_inputs('new_user', login_page)
	60
	61	login_form.update({
	62	'user[login]': username,
	63	'user[password]': password,
	64	})
	65
	66	post_url = self._search_regex(
	67	r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page,
	68	'post url', default=self._LOGIN_URL, group='url')
	69
	70	if not post_url.startswith('http'):
	71	post_url = compat_urlparse.urljoin(self._LOGIN_URL, post_url)
	72
	73	request = sanitized_Request(
	74	post_url, urlencode_postdata(encode_dict(login_form)))
	75	request.add_header('Referer', self._LOGIN_URL)
	76
	77	response = self._download_webpage(
	78	request, None, 'Logging in as %s' % username)
	79
	80	if all(p not in response for p in ('>Logout<', 'href="/users/sign_out"')):
	81	error = self._search_regex(
	82	r'<p class="alert alert-danger">(.+?)</p>',
	83	response, 'error', default=None)
	84	if error:
	85	raise ExtractorError('Unable to login: %s' % error, expected=True)
	86	raise ExtractorError('Unable to log in')
	87
	88	def _real_initialize(self):
	89	self._login()
	90
	91	def _real_extract(self, url):
	92	anime_id = self._match_id(url)
	93
	94	webpage = self._download_webpage(url, anime_id)
	95
	96	if 'data-playlist=' not in webpage:
	97	self._download_webpage(
	98	self._APPLY_HTML5_URL, anime_id,
	99	'Activating HTML5 beta', 'Unable to apply HTML5 beta')
	100	webpage = self._download_webpage(url, anime_id)
	101
	102	csrf_token = self._html_search_meta(
	103	'csrf-token', webpage, 'csrf token', fatal=True)
	104
	105	anime_title = self._html_search_regex(
	106	r'(?s)<h1[^>]+itemprop="name"[^>]*>(.+?)</h1>',
	107	webpage, 'anime name')
	108	anime_description = self._html_search_regex(
	109	r'(?s)<div[^>]+itemprop="description"[^>]*>(.+?)</div>',
	110	webpage, 'anime description', default=None)
	111
	112	entries = []
	113
85e8f26b	114	for num, episode_html in enumerate(re.findall(
0d0e2829	115	r'(?s)<h3[^>]+class="episodebox-title".+?>Episodeninhalt<', webpage), 1):
85e8f26b	116	episodebox_title = self._search_regex(
0d0e2829 S	117	(r'class="episodebox-title"[^>]+title=(["\'])(?P<title>.+?)\1',
	118	r'class="episodebox-title"[^>]+>(?P<title>.+?)<'),
	119	episode_html, 'episodebox title', default=None, group='title')
85e8f26b	120	if not episodebox_title:
e2bd68c9 S	121	continue
e2bd68c9 S	122
85e8f26b	123	episode_number = int(self._search_regex(
0d0e2829	124	r'(?:Episode\|Film)\s*(\d+)',
85e8f26b S	125	episodebox_title, 'episode number', default=num))
85e8f26b S	126	episode_title = self._search_regex(
0d0e2829	127	r'(?:Episode\|Film)\s\d+\s-\s*(.+)',
85e8f26b S	128	episodebox_title, 'episode title', default=None)
85e8f26b S	129
e2bd68c9 S	130	video_id = 'episode-%d' % episode_number
	131
	132	common_info = {
	133	'id': video_id,
	134	'series': anime_title,
	135	'episode': episode_title,
	136	'episode_number': episode_number,
	137	}
	138
	139	formats = []
	140
3c5d183c S	141	for input_ in re.findall(
	142	r'<input[^>]+class=["\'].*?streamstarter_html5[^>]+>', episode_html):
	143	attributes = extract_attributes(input_)
	144	playlist_urls = []
	145	for playlist_key in ('data-playlist', 'data-otherplaylist'):
	146	playlist_url = attributes.get(playlist_key)
	147	if isinstance(playlist_url, compat_str) and re.match(
	148	r'/?[\da-zA-Z]+', playlist_url):
	149	playlist_urls.append(attributes[playlist_key])
	150	if not playlist_urls:
	151	continue
	152
	153	lang = attributes.get('data-lang')
	154	lang_note = attributes.get('value')
	155
	156	for playlist_url in playlist_urls:
	157	kind = self._search_regex(
	158	r'videomaterialurl/\d+/([^/]+)/',
	159	playlist_url, 'media kind', default=None)
	160	format_id_list = []
	161	if lang:
	162	format_id_list.append(lang)
	163	if kind:
	164	format_id_list.append(kind)
	165	if not format_id_list:
	166	format_id_list.append('hls')
	167	format_id = '-'.join(format_id_list)
	168	format_note = ', '.join(filter(None, (kind, lang_note)))
	169	request = sanitized_Request(
	170	compat_urlparse.urljoin(url, playlist_url),
	171	headers={
	172	'X-Requested-With': 'XMLHttpRequest',
	173	'X-CSRF-Token': csrf_token,
	174	'Referer': url,
	175	'Accept': 'application/json, text/javascript, /; q=0.01',
	176	})
	177	playlist = self._download_json(
	178	request, video_id, 'Downloading %s playlist JSON' % format_id,
	179	fatal=False)
	180	if not playlist:
	181	continue
	182	playlist = playlist.get('playlist')
	183	if not playlist or not isinstance(playlist, list):
	184	continue
	185	playlist = playlist[0]
	186	title = playlist.get('title')
	187	if not title:
	188	continue
e2bd68c9 S	189	description = playlist.get('description')
	190	for source in playlist.get('sources', []):
	191	file_ = source.get('file')
	192	if file_ and determine_ext(file_) == 'm3u8':
3c5d183c	193	m3u8_formats = self._extract_m3u8_formats(
e2bd68c9	194	file_, video_id, 'mp4',
3c5d183c S	195	entry_protocol='m3u8_native', m3u8_id=format_id)
	196	for f in m3u8_formats:
	197	f.update({
	198	'language': lang,
	199	'format_note': format_note,
	200	})
	201	formats.extend(m3u8_formats)
e2bd68c9 S	202
e2bd68c9 S	203	if formats:
3c5d183c	204	self._sort_formats(formats)
e2bd68c9 S	205	f = common_info.copy()
	206	f.update({
	207	'title': title,
	208	'description': description,
	209	'formats': formats,
	210	})
	211	entries.append(f)
	212
	213	m = re.search(
	214	r'data-dialog-header=(["\'])(?P<title>.+?)\1[^>]+href=(["\'])(?P<href>.+?)\3[^>]*>Teaser<',
	215	episode_html)
	216	if m:
	217	f = common_info.copy()
	218	f.update({
	219	'id': '%s-teaser' % f['id'],
	220	'title': m.group('title'),
	221	'url': compat_urlparse.urljoin(url, m.group('href')),
	222	})
	223	entries.append(f)
	224
	225	return self.playlist_result(entries, anime_id, anime_title, anime_description)