yt_dlp/extractor/animeondemand.py

   1 import re
   2
   3 from .common import InfoExtractor
   4 from ..compat import compat_str
   5 from ..utils import (
   6     determine_ext,
   7     extract_attributes,
   8     ExtractorError,
   9     join_nonempty,
  10     url_or_none,
  11     urlencode_postdata,
  12     urljoin,
  13 )
  14
  15
  16 class AnimeOnDemandIE(InfoExtractor):
  17     _VALID_URL = r'https?://(?:www\.)?anime-on-demand\.de/anime/(?P<id>\d+)'
  18     _LOGIN_URL = 'https://www.anime-on-demand.de/users/sign_in'
  19     _APPLY_HTML5_URL = 'https://www.anime-on-demand.de/html5apply'
  20     _NETRC_MACHINE = 'animeondemand'
  21     # German-speaking countries of Europe
  22     _GEO_COUNTRIES = ['AT', 'CH', 'DE', 'LI', 'LU']
  23     _TESTS = [{
  24         # jap, OmU
  25         'url': 'https://www.anime-on-demand.de/anime/161',
  26         'info_dict': {
  27             'id': '161',
  28             'title': 'Grimgar, Ashes and Illusions (OmU)',
  29             'description': 'md5:6681ce3c07c7189d255ac6ab23812d31',
  30         },
  31         'playlist_mincount': 4,
  32     }, {
  33         # Film wording is used instead of Episode, ger/jap, Dub/OmU
  34         'url': 'https://www.anime-on-demand.de/anime/39',
  35         'only_matching': True,
  36     }, {
  37         # Episodes without titles, jap, OmU
  38         'url': 'https://www.anime-on-demand.de/anime/162',
  39         'only_matching': True,
  40     }, {
  41         # ger/jap, Dub/OmU, account required
  42         'url': 'https://www.anime-on-demand.de/anime/169',
  43         'only_matching': True,
  44     }, {
  45         # Full length film, non-series, ger/jap, Dub/OmU, account required
  46         'url': 'https://www.anime-on-demand.de/anime/185',
  47         'only_matching': True,
  48     }, {
  49         # Flash videos
  50         'url': 'https://www.anime-on-demand.de/anime/12',
  51         'only_matching': True,
  52     }]
  53
  54     def _perform_login(self, username, password):
  55         login_page = self._download_webpage(
  56             self._LOGIN_URL, None, 'Downloading login page')
  57
  58         if '>Our licensing terms allow the distribution of animes only to German-speaking countries of Europe' in login_page:
  59             self.raise_geo_restricted(
  60                 '%s is only available in German-speaking countries of Europe' % self.IE_NAME)
  61
  62         login_form = self._form_hidden_inputs('new_user', login_page)
  63
  64         login_form.update({
  65             'user[login]': username,
  66             'user[password]': password,
  67         })
  68
  69         post_url = self._search_regex(
  70             r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page,
  71             'post url', default=self._LOGIN_URL, group='url')
  72
  73         if not post_url.startswith('http'):
  74             post_url = urljoin(self._LOGIN_URL, post_url)
  75
  76         response = self._download_webpage(
  77             post_url, None, 'Logging in',
  78             data=urlencode_postdata(login_form), headers={
  79                 'Referer': self._LOGIN_URL,
  80             })
  81
  82         if all(p not in response for p in ('>Logout<', 'href="/users/sign_out"')):
  83             error = self._search_regex(
  84                 r'<p[^>]+\bclass=(["\'])(?:(?!\1).)*\balert\b(?:(?!\1).)*\1[^>]*>(?P<error>.+?)</p>',
  85                 response, 'error', default=None, group='error')
  86             if error:
  87                 raise ExtractorError('Unable to login: %s' % error, expected=True)
  88             raise ExtractorError('Unable to log in')
  89
  90     def _real_extract(self, url):
  91         anime_id = self._match_id(url)
  92
  93         webpage = self._download_webpage(url, anime_id)
  94
  95         if 'data-playlist=' not in webpage:
  96             self._download_webpage(
  97                 self._APPLY_HTML5_URL, anime_id,
  98                 'Activating HTML5 beta', 'Unable to apply HTML5 beta')
  99             webpage = self._download_webpage(url, anime_id)
 100
 101         csrf_token = self._html_search_meta(
 102             'csrf-token', webpage, 'csrf token', fatal=True)
 103
 104         anime_title = self._html_search_regex(
 105             r'(?s)<h1[^>]+itemprop="name"[^>]*>(.+?)</h1>',
 106             webpage, 'anime name')
 107         anime_description = self._html_search_regex(
 108             r'(?s)<div[^>]+itemprop="description"[^>]*>(.+?)</div>',
 109             webpage, 'anime description', default=None)
 110
 111         def extract_info(html, video_id, num=None):
 112             title, description = [None] * 2
 113             formats = []
 114
 115             for input_ in re.findall(
 116                     r'<input[^>]+class=["\'].*?streamstarter[^>]+>', html):
 117                 attributes = extract_attributes(input_)
 118                 title = attributes.get('data-dialog-header')
 119                 playlist_urls = []
 120                 for playlist_key in ('data-playlist', 'data-otherplaylist', 'data-stream'):
 121                     playlist_url = attributes.get(playlist_key)
 122                     if isinstance(playlist_url, compat_str) and re.match(
 123                             r'/?[\da-zA-Z]+', playlist_url):
 124                         playlist_urls.append(attributes[playlist_key])
 125                 if not playlist_urls:
 126                     continue
 127
 128                 lang = attributes.get('data-lang')
 129                 lang_note = attributes.get('value')
 130
 131                 for playlist_url in playlist_urls:
 132                     kind = self._search_regex(
 133                         r'videomaterialurl/\d+/([^/]+)/',
 134                         playlist_url, 'media kind', default=None)
 135                     format_id = join_nonempty(lang, kind) if lang or kind else str(num)
 136                     format_note = join_nonempty(kind, lang_note, delim=', ')
 137                     item_id_list = []
 138                     if format_id:
 139                         item_id_list.append(format_id)
 140                     item_id_list.append('videomaterial')
 141                     playlist = self._download_json(
 142                         urljoin(url, playlist_url), video_id,
 143                         'Downloading %s JSON' % ' '.join(item_id_list),
 144                         headers={
 145                             'X-Requested-With': 'XMLHttpRequest',
 146                             'X-CSRF-Token': csrf_token,
 147                             'Referer': url,
 148                             'Accept': 'application/json, text/javascript, */*; q=0.01',
 149                         }, fatal=False)
 150                     if not playlist:
 151                         continue
 152                     stream_url = url_or_none(playlist.get('streamurl'))
 153                     if stream_url:
 154                         rtmp = re.search(
 155                             r'^(?P<url>rtmpe?://(?P<host>[^/]+)/(?P<app>.+/))(?P<playpath>mp[34]:.+)',
 156                             stream_url)
 157                         if rtmp:
 158                             formats.append({
 159                                 'url': rtmp.group('url'),
 160                                 'app': rtmp.group('app'),
 161                                 'play_path': rtmp.group('playpath'),
 162                                 'page_url': url,
 163                                 'player_url': 'https://www.anime-on-demand.de/assets/jwplayer.flash-55abfb34080700304d49125ce9ffb4a6.swf',
 164                                 'rtmp_real_time': True,
 165                                 'format_id': 'rtmp',
 166                                 'ext': 'flv',
 167                             })
 168                             continue
 169                     start_video = playlist.get('startvideo', 0)
 170                     playlist = playlist.get('playlist')
 171                     if not playlist or not isinstance(playlist, list):
 172                         continue
 173                     playlist = playlist[start_video]
 174                     title = playlist.get('title')
 175                     if not title:
 176                         continue
 177                     description = playlist.get('description')
 178                     for source in playlist.get('sources', []):
 179                         file_ = source.get('file')
 180                         if not file_:
 181                             continue
 182                         ext = determine_ext(file_)
 183                         format_id = join_nonempty(
 184                             lang, kind,
 185                             'hls' if ext == 'm3u8' else None,
 186                             'dash' if source.get('type') == 'video/dash' or ext == 'mpd' else None)
 187                         if ext == 'm3u8':
 188                             file_formats = self._extract_m3u8_formats(
 189                                 file_, video_id, 'mp4',
 190                                 entry_protocol='m3u8_native', m3u8_id=format_id, fatal=False)
 191                         elif source.get('type') == 'video/dash' or ext == 'mpd':
 192                             continue
 193                             file_formats = self._extract_mpd_formats(
 194                                 file_, video_id, mpd_id=format_id, fatal=False)
 195                         else:
 196                             continue
 197                         for f in file_formats:
 198                             f.update({
 199                                 'language': lang,
 200                                 'format_note': format_note,
 201                             })
 202                         formats.extend(file_formats)
 203
 204             return {
 205                 'title': title,
 206                 'description': description,
 207                 'formats': formats,
 208             }
 209
 210         def extract_entries(html, video_id, common_info, num=None):
 211             info = extract_info(html, video_id, num)
 212
 213             if info['formats']:
 214                 self._sort_formats(info['formats'])
 215                 f = common_info.copy()
 216                 f.update(info)
 217                 yield f
 218
 219             # Extract teaser/trailer only when full episode is not available
 220             if not info['formats']:
 221                 m = re.search(
 222                     r'data-dialog-header=(["\'])(?P<title>.+?)\1[^>]+href=(["\'])(?P<href>.+?)\3[^>]*>(?P<kind>Teaser|Trailer)<',
 223                     html)
 224                 if m:
 225                     f = common_info.copy()
 226                     f.update({
 227                         'id': '%s-%s' % (f['id'], m.group('kind').lower()),
 228                         'title': m.group('title'),
 229                         'url': urljoin(url, m.group('href')),
 230                     })
 231                     yield f
 232
 233         def extract_episodes(html):
 234             for num, episode_html in enumerate(re.findall(
 235                     r'(?s)<h3[^>]+class="episodebox-title".+?>Episodeninhalt<', html), 1):
 236                 episodebox_title = self._search_regex(
 237                     (r'class="episodebox-title"[^>]+title=(["\'])(?P<title>.+?)\1',
 238                      r'class="episodebox-title"[^>]+>(?P<title>.+?)<'),
 239                     episode_html, 'episodebox title', default=None, group='title')
 240                 if not episodebox_title:
 241                     continue
 242
 243                 episode_number = int(self._search_regex(
 244                     r'(?:Episode|Film)\s*(\d+)',
 245                     episodebox_title, 'episode number', default=num))
 246                 episode_title = self._search_regex(
 247                     r'(?:Episode|Film)\s*\d+\s*-\s*(.+)',
 248                     episodebox_title, 'episode title', default=None)
 249
 250                 video_id = 'episode-%d' % episode_number
 251
 252                 common_info = {
 253                     'id': video_id,
 254                     'series': anime_title,
 255                     'episode': episode_title,
 256                     'episode_number': episode_number,
 257                 }
 258
 259                 for e in extract_entries(episode_html, video_id, common_info):
 260                     yield e
 261
 262         def extract_film(html, video_id):
 263             common_info = {
 264                 'id': anime_id,
 265                 'title': anime_title,
 266                 'description': anime_description,
 267             }
 268             for e in extract_entries(html, video_id, common_info):
 269                 yield e
 270
 271         def entries():
 272             has_episodes = False
 273             for e in extract_episodes(webpage):
 274                 has_episodes = True
 275                 yield e
 276
 277             if not has_episodes:
 278                 for e in extract_film(webpage, anime_id):
 279                     yield e
 280
 281         return self.playlist_result(
 282             entries(), anime_id, anime_title, anime_description)