yt_dlp/extractor/funimation.py

   1 # coding: utf-8
   2 from __future__ import unicode_literals
   3
   4 import random
   5 import re
   6 import string
   7
   8 from .common import InfoExtractor
   9 from ..compat import compat_HTTPError
  10 from ..utils import (
  11     determine_ext,
  12     int_or_none,
  13     join_nonempty,
  14     js_to_json,
  15     orderedSet,
  16     qualities,
  17     str_or_none,
  18     traverse_obj,
  19     try_get,
  20     urlencode_postdata,
  21     ExtractorError,
  22 )
  23
  24
  25 class FunimationBaseIE(InfoExtractor):
  26     _NETRC_MACHINE = 'funimation'
  27     _REGION = None
  28     _TOKEN = None
  29
  30     def _get_region(self):
  31         region_cookie = self._get_cookies('https://www.funimation.com').get('region')
  32         region = region_cookie.value if region_cookie else self.get_param('geo_bypass_country')
  33         return region or traverse_obj(
  34             self._download_json(
  35                 'https://geo-service.prd.funimationsvc.com/geo/v1/region/check', None, fatal=False,
  36                 note='Checking geo-location', errnote='Unable to fetch geo-location information'),
  37             'region') or 'US'
  38
  39     def _login(self):
  40         username, password = self._get_login_info()
  41         if username is None:
  42             return
  43         try:
  44             data = self._download_json(
  45                 'https://prod-api-funimationnow.dadcdigital.com/api/auth/login/',
  46                 None, 'Logging in', data=urlencode_postdata({
  47                     'username': username,
  48                     'password': password,
  49                 }))
  50             return data['token']
  51         except ExtractorError as e:
  52             if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
  53                 error = self._parse_json(e.cause.read().decode(), None)['error']
  54                 raise ExtractorError(error, expected=True)
  55             raise
  56
  57
  58 class FunimationPageIE(FunimationBaseIE):
  59     IE_NAME = 'funimation:page'
  60     _VALID_URL = r'https?://(?:www\.)?funimation(?:\.com|now\.uk)/(?:(?P<lang>[^/]+)/)?(?:shows|v)/(?P<show>[^/]+)/(?P<episode>[^/?#&]+)'
  61
  62     _TESTS = [{
  63         'url': 'https://www.funimation.com/shows/attack-on-titan-junior-high/broadcast-dub-preview/',
  64         'info_dict': {
  65             'id': '210050',
  66             'ext': 'mp4',
  67             'title': 'Broadcast Dub Preview',
  68             # Other metadata is tested in FunimationIE
  69         },
  70         'params': {
  71             'skip_download': 'm3u8',
  72         },
  73         'add_ie': ['Funimation'],
  74     }, {
  75         # Not available in US
  76         'url': 'https://www.funimation.com/shows/hacksign/role-play/',
  77         'only_matching': True,
  78     }, {
  79         # with lang code
  80         'url': 'https://www.funimation.com/en/shows/hacksign/role-play/',
  81         'only_matching': True,
  82     }, {
  83         'url': 'https://www.funimationnow.uk/shows/puzzle-dragons-x/drop-impact/simulcast/',
  84         'only_matching': True,
  85     }, {
  86         'url': 'https://www.funimation.com/v/a-certain-scientific-railgun/super-powered-level-5',
  87         'only_matching': True,
  88     }]
  89
  90     def _real_initialize(self):
  91         if not self._REGION:
  92             FunimationBaseIE._REGION = self._get_region()
  93         if not self._TOKEN:
  94             FunimationBaseIE._TOKEN = self._login()
  95
  96     def _real_extract(self, url):
  97         locale, show, episode = self._match_valid_url(url).group('lang', 'show', 'episode')
  98
  99         video_id = traverse_obj(self._download_json(
 100             f'https://title-api.prd.funimationsvc.com/v1/shows/{show}/episodes/{episode}',
 101             f'{show}_{episode}', query={
 102                 'deviceType': 'web',
 103                 'region': self._REGION,
 104                 'locale': locale or 'en'
 105             }), ('videoList', ..., 'id'), get_all=False)
 106
 107         return self.url_result(f'https://www.funimation.com/player/{video_id}', FunimationIE.ie_key(), video_id)
 108
 109
 110 class FunimationIE(FunimationBaseIE):
 111     _VALID_URL = r'https?://(?:www\.)?funimation\.com/player/(?P<id>\d+)'
 112
 113     _TESTS = [{
 114         'url': 'https://www.funimation.com/player/210051',
 115         'info_dict': {
 116             'id': '210050',
 117             'display_id': 'broadcast-dub-preview',
 118             'ext': 'mp4',
 119             'title': 'Broadcast Dub Preview',
 120             'thumbnail': r're:https?://.*\.(?:jpg|png)',
 121             'episode': 'Broadcast Dub Preview',
 122             'episode_id': '210050',
 123             'season': 'Extras',
 124             'season_id': '166038',
 125             'season_number': 99,
 126             'series': 'Attack on Titan: Junior High',
 127             'description': '',
 128             'duration': 155,
 129         },
 130         'params': {
 131             'skip_download': 'm3u8',
 132         },
 133     }, {
 134         'note': 'player_id should be extracted with the relevent compat-opt',
 135         'url': 'https://www.funimation.com/player/210051',
 136         'info_dict': {
 137             'id': '210051',
 138             'display_id': 'broadcast-dub-preview',
 139             'ext': 'mp4',
 140             'title': 'Broadcast Dub Preview',
 141             'thumbnail': r're:https?://.*\.(?:jpg|png)',
 142             'episode': 'Broadcast Dub Preview',
 143             'episode_id': '210050',
 144             'season': 'Extras',
 145             'season_id': '166038',
 146             'season_number': 99,
 147             'series': 'Attack on Titan: Junior High',
 148             'description': '',
 149             'duration': 155,
 150         },
 151         'params': {
 152             'skip_download': 'm3u8',
 153             'compat_opts': ['seperate-video-versions'],
 154         },
 155     }]
 156
 157     def _real_initialize(self):
 158         if not self._TOKEN:
 159             FunimationBaseIE._TOKEN = self._login()
 160
 161     @staticmethod
 162     def _get_experiences(episode):
 163         for lang, lang_data in episode.get('languages', {}).items():
 164             for video_data in lang_data.values():
 165                 for version, f in video_data.items():
 166                     yield lang, version.title(), f
 167
 168     def _get_episode(self, webpage, experience_id=None, episode_id=None, fatal=True):
 169         ''' Extract the episode, season and show objects given either episode/experience id '''
 170         show = self._parse_json(
 171             self._search_regex(
 172                 r'show\s*=\s*({.+?})\s*;', webpage, 'show data', fatal=fatal),
 173             experience_id, transform_source=js_to_json, fatal=fatal) or []
 174         for season in show.get('seasons', []):
 175             for episode in season.get('episodes', []):
 176                 if episode_id is not None:
 177                     if str(episode.get('episodePk')) == episode_id:
 178                         return episode, season, show
 179                     continue
 180                 for _, _, f in self._get_experiences(episode):
 181                     if f.get('experienceId') == experience_id:
 182                         return episode, season, show
 183         if fatal:
 184             raise ExtractorError('Unable to find episode information')
 185         else:
 186             self.report_warning('Unable to find episode information')
 187         return {}, {}, {}
 188
 189     def _real_extract(self, url):
 190         initial_experience_id = self._match_id(url)
 191         webpage = self._download_webpage(
 192             url, initial_experience_id, note=f'Downloading player webpage for {initial_experience_id}')
 193         episode, season, show = self._get_episode(webpage, experience_id=int(initial_experience_id))
 194         episode_id = str(episode['episodePk'])
 195         display_id = episode.get('slug') or episode_id
 196
 197         formats, subtitles, thumbnails, duration = [], {}, [], 0
 198         requested_languages, requested_versions = self._configuration_arg('language'), self._configuration_arg('version')
 199         language_preference = qualities((requested_languages or [''])[::-1])
 200         source_preference = qualities((requested_versions or ['uncut', 'simulcast'])[::-1])
 201         only_initial_experience = 'seperate-video-versions' in self.get_param('compat_opts', [])
 202
 203         for lang, version, fmt in self._get_experiences(episode):
 204             experience_id = str(fmt['experienceId'])
 205             if (only_initial_experience and experience_id != initial_experience_id
 206                     or requested_languages and lang.lower() not in requested_languages
 207                     or requested_versions and version.lower() not in requested_versions):
 208                 continue
 209             thumbnails.append({'url': fmt.get('poster')})
 210             duration = max(duration, fmt.get('duration', 0))
 211             format_name = '%s %s (%s)' % (version, lang, experience_id)
 212             self.extract_subtitles(
 213                 subtitles, experience_id, display_id=display_id, format_name=format_name,
 214                 episode=episode if experience_id == initial_experience_id else episode_id)
 215
 216             headers = {}
 217             if self._TOKEN:
 218                 headers['Authorization'] = 'Token %s' % self._TOKEN
 219             page = self._download_json(
 220                 'https://www.funimation.com/api/showexperience/%s/' % experience_id,
 221                 display_id, headers=headers, expected_status=403, query={
 222                     'pinst_id': ''.join([random.choice(string.digits + string.ascii_letters) for _ in range(8)]),
 223                 }, note=f'Downloading {format_name} JSON')
 224             sources = page.get('items') or []
 225             if not sources:
 226                 error = try_get(page, lambda x: x['errors'][0], dict)
 227                 if error:
 228                     self.report_warning('%s said: Error %s - %s' % (
 229                         self.IE_NAME, error.get('code'), error.get('detail') or error.get('title')))
 230                 else:
 231                     self.report_warning('No sources found for format')
 232
 233             current_formats = []
 234             for source in sources:
 235                 source_url = source.get('src')
 236                 source_type = source.get('videoType') or determine_ext(source_url)
 237                 if source_type == 'm3u8':
 238                     current_formats.extend(self._extract_m3u8_formats(
 239                         source_url, display_id, 'mp4', m3u8_id='%s-%s' % (experience_id, 'hls'), fatal=False,
 240                         note=f'Downloading {format_name} m3u8 information'))
 241                 else:
 242                     current_formats.append({
 243                         'format_id': '%s-%s' % (experience_id, source_type),
 244                         'url': source_url,
 245                     })
 246                 for f in current_formats:
 247                     # TODO: Convert language to code
 248                     f.update({
 249                         'language': lang,
 250                         'format_note': version,
 251                         'source_preference': source_preference(version.lower()),
 252                         'language_preference': language_preference(lang.lower()),
 253                     })
 254                 formats.extend(current_formats)
 255         self._remove_duplicate_formats(formats)
 256         self._sort_formats(formats, ('lang', 'source'))
 257
 258         return {
 259             'id': initial_experience_id if only_initial_experience else episode_id,
 260             'display_id': display_id,
 261             'duration': duration,
 262             'title': episode['episodeTitle'],
 263             'description': episode.get('episodeSummary'),
 264             'episode': episode.get('episodeTitle'),
 265             'episode_number': int_or_none(episode.get('episodeId')),
 266             'episode_id': episode_id,
 267             'season': season.get('seasonTitle'),
 268             'season_number': int_or_none(season.get('seasonId')),
 269             'season_id': str_or_none(season.get('seasonPk')),
 270             'series': show.get('showTitle'),
 271             'formats': formats,
 272             'thumbnails': thumbnails,
 273             'subtitles': subtitles,
 274         }
 275
 276     def _get_subtitles(self, subtitles, experience_id, episode, display_id, format_name):
 277         if isinstance(episode, str):
 278             webpage = self._download_webpage(
 279                 f'https://www.funimation.com/player/{experience_id}', display_id,
 280                 fatal=False, note=f'Downloading player webpage for {format_name}')
 281             episode, _, _ = self._get_episode(webpage, episode_id=episode, fatal=False)
 282
 283         for _, version, f in self._get_experiences(episode):
 284             for source in f.get('sources'):
 285                 for text_track in source.get('textTracks'):
 286                     if not text_track.get('src'):
 287                         continue
 288                     sub_type = text_track.get('type').upper()
 289                     sub_type = sub_type if sub_type != 'FULL' else None
 290                     current_sub = {
 291                         'url': text_track['src'],
 292                         'name': join_nonempty(version, text_track.get('label'), sub_type, delim=' ')
 293                     }
 294                     lang = join_nonempty(text_track.get('language', 'und'),
 295                                          version if version != 'Simulcast' else None,
 296                                          sub_type, delim='_')
 297                     if current_sub not in subtitles.get(lang, []):
 298                         subtitles.setdefault(lang, []).append(current_sub)
 299         return subtitles
 300
 301
 302 class FunimationShowIE(FunimationBaseIE):
 303     IE_NAME = 'funimation:show'
 304     _VALID_URL = r'(?P<url>https?://(?:www\.)?funimation(?:\.com|now\.uk)/(?P<locale>[^/]+)?/?shows/(?P<id>[^/?#&]+))/?(?:[?#]|$)'
 305
 306     _TESTS = [{
 307         'url': 'https://www.funimation.com/en/shows/sk8-the-infinity',
 308         'info_dict': {
 309             'id': 1315000,
 310             'title': 'SK8 the Infinity'
 311         },
 312         'playlist_count': 13,
 313         'params': {
 314             'skip_download': True,
 315         },
 316     }, {
 317         # without lang code
 318         'url': 'https://www.funimation.com/shows/ouran-high-school-host-club/',
 319         'info_dict': {
 320             'id': 39643,
 321             'title': 'Ouran High School Host Club'
 322         },
 323         'playlist_count': 26,
 324         'params': {
 325             'skip_download': True,
 326         },
 327     }]
 328
 329     def _real_initialize(self):
 330         if not self._REGION:
 331             FunimationBaseIE._REGION = self._get_region()
 332
 333     def _real_extract(self, url):
 334         base_url, locale, display_id = self._match_valid_url(url).groups()
 335
 336         show_info = self._download_json(
 337             'https://title-api.prd.funimationsvc.com/v2/shows/%s?region=%s&deviceType=web&locale=%s'
 338             % (display_id, self._REGION, locale or 'en'), display_id)
 339         items_info = self._download_json(
 340             'https://prod-api-funimationnow.dadcdigital.com/api/funimation/episodes/?limit=99999&title_id=%s'
 341             % show_info.get('id'), display_id)
 342
 343         vod_items = traverse_obj(items_info, ('items', ..., re.compile('(?i)mostRecent[AS]vod').match, 'item'))
 344
 345         return {
 346             '_type': 'playlist',
 347             'id': show_info['id'],
 348             'title': show_info['name'],
 349             'entries': orderedSet(
 350                 self.url_result(
 351                     '%s/%s' % (base_url, vod_item.get('episodeSlug')), FunimationPageIE.ie_key(),
 352                     vod_item.get('episodeId'), vod_item.get('episodeName'))
 353                 for vod_item in sorted(vod_items, key=lambda x: x.get('episodeOrder', -1))),
 354         }