yt_dlp/extractor/funimation.py

   1 # coding: utf-8
   2 from __future__ import unicode_literals
   3
   4 import random
   5 import string
   6
   7 from .common import InfoExtractor
   8 from ..compat import compat_HTTPError
   9 from ..utils import (
  10     determine_ext,
  11     dict_get,
  12     int_or_none,
  13     js_to_json,
  14     str_or_none,
  15     try_get,
  16     qualities,
  17     urlencode_postdata,
  18     ExtractorError,
  19 )
  20
  21
  22 class FunimationPageIE(InfoExtractor):
  23     IE_NAME = 'funimation:page'
  24     _VALID_URL = r'(?P<origin>https?://(?:www\.)?funimation(?:\.com|now\.uk))/(?P<lang>[^/]+/)?(?P<path>shows/(?P<id>[^/]+/[^/?#&]+).*$)'
  25
  26     _TESTS = [{
  27         'url': 'https://www.funimation.com/shows/attack-on-titan-junior-high/broadcast-dub-preview/',
  28         'info_dict': {
  29             'id': '210050',
  30             'ext': 'mp4',
  31             'title': 'Broadcast Dub Preview',
  32             # Other metadata is tested in FunimationIE
  33         },
  34         'params': {
  35             'skip_download': 'm3u8',
  36         },
  37         'add_ie': ['Funimation'],
  38     }, {
  39         # Not available in US
  40         'url': 'https://www.funimation.com/shows/hacksign/role-play/',
  41         'only_matching': True,
  42     }, {
  43         # with lang code
  44         'url': 'https://www.funimation.com/en/shows/hacksign/role-play/',
  45         'only_matching': True,
  46     }, {
  47         'url': 'https://www.funimationnow.uk/shows/puzzle-dragons-x/drop-impact/simulcast/',
  48         'only_matching': True,
  49     }]
  50
  51     def _real_extract(self, url):
  52         mobj = self._match_valid_url(url)
  53         display_id = mobj.group('id').replace('/', '_')
  54         if not mobj.group('lang'):
  55             url = '%s/en/%s' % (mobj.group('origin'), mobj.group('path'))
  56
  57         webpage = self._download_webpage(url, display_id)
  58         title_data = self._parse_json(self._search_regex(
  59             r'TITLE_DATA\s*=\s*({[^}]+})',
  60             webpage, 'title data', default=''),
  61             display_id, js_to_json, fatal=False) or {}
  62
  63         video_id = (
  64             title_data.get('id')
  65             or self._search_regex(
  66                 (r"KANE_customdimensions.videoID\s*=\s*'(\d+)';", r'<iframe[^>]+src="/player/(\d+)'),
  67                 webpage, 'video_id', default=None)
  68             or self._search_regex(
  69                 r'/player/(\d+)',
  70                 self._html_search_meta(['al:web:url', 'og:video:url', 'og:video:secure_url'], webpage, fatal=True),
  71                 'video id'))
  72         return self.url_result(f'https://www.funimation.com/player/{video_id}', FunimationIE.ie_key(), video_id)
  73
  74
  75 class FunimationIE(InfoExtractor):
  76     _VALID_URL = r'https?://(?:www\.)?funimation\.com/player/(?P<id>\d+)'
  77
  78     _NETRC_MACHINE = 'funimation'
  79     _TOKEN = None
  80
  81     _TESTS = [{
  82         'url': 'https://www.funimation.com/player/210051',
  83         'info_dict': {
  84             'id': '210050',
  85             'display_id': 'broadcast-dub-preview',
  86             'ext': 'mp4',
  87             'title': 'Broadcast Dub Preview',
  88             'thumbnail': r're:https?://.*\.(?:jpg|png)',
  89             'episode': 'Broadcast Dub Preview',
  90             'episode_id': '210050',
  91             'season': 'Extras',
  92             'season_id': '166038',
  93             'season_number': 99,
  94             'series': 'Attack on Titan: Junior High',
  95             'description': '',
  96             'duration': 154,
  97         },
  98         'params': {
  99             'skip_download': 'm3u8',
 100         },
 101     }, {
 102         'note': 'player_id should be extracted with the relevent compat-opt',
 103         'url': 'https://www.funimation.com/player/210051',
 104         'info_dict': {
 105             'id': '210051',
 106             'display_id': 'broadcast-dub-preview',
 107             'ext': 'mp4',
 108             'title': 'Broadcast Dub Preview',
 109             'thumbnail': r're:https?://.*\.(?:jpg|png)',
 110             'episode': 'Broadcast Dub Preview',
 111             'episode_id': '210050',
 112             'season': 'Extras',
 113             'season_id': '166038',
 114             'season_number': 99,
 115             'series': 'Attack on Titan: Junior High',
 116             'description': '',
 117             'duration': 154,
 118         },
 119         'params': {
 120             'skip_download': 'm3u8',
 121             'compat_opts': ['seperate-video-versions'],
 122         },
 123     }]
 124
 125     def _login(self):
 126         username, password = self._get_login_info()
 127         if username is None:
 128             return
 129         try:
 130             data = self._download_json(
 131                 'https://prod-api-funimationnow.dadcdigital.com/api/auth/login/',
 132                 None, 'Logging in', data=urlencode_postdata({
 133                     'username': username,
 134                     'password': password,
 135                 }))
 136             self._TOKEN = data['token']
 137         except ExtractorError as e:
 138             if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
 139                 error = self._parse_json(e.cause.read().decode(), None)['error']
 140                 raise ExtractorError(error, expected=True)
 141             raise
 142
 143     def _real_initialize(self):
 144         self._login()
 145
 146     @staticmethod
 147     def _get_experiences(episode):
 148         for lang, lang_data in episode.get('languages', {}).items():
 149             for video_data in lang_data.values():
 150                 for version, f in video_data.items():
 151                     yield lang, version.title(), f
 152
 153     def _get_episode(self, webpage, experience_id=None, episode_id=None, fatal=True):
 154         ''' Extract the episode, season and show objects given either episode/experience id '''
 155         show = self._parse_json(
 156             self._search_regex(
 157                 r'show\s*=\s*({.+?})\s*;', webpage, 'show data', fatal=fatal),
 158             experience_id, transform_source=js_to_json, fatal=fatal) or []
 159         for season in show.get('seasons', []):
 160             for episode in season.get('episodes', []):
 161                 if episode_id is not None:
 162                     if str(episode.get('episodePk')) == episode_id:
 163                         return episode, season, show
 164                     continue
 165                 for _, _, f in self._get_experiences(episode):
 166                     if f.get('experienceId') == experience_id:
 167                         return episode, season, show
 168         if fatal:
 169             raise ExtractorError('Unable to find episode information')
 170         else:
 171             self.report_warning('Unable to find episode information')
 172         return {}, {}, {}
 173
 174     def _real_extract(self, url):
 175         initial_experience_id = self._match_id(url)
 176         webpage = self._download_webpage(
 177             url, initial_experience_id, note=f'Downloading player webpage for {initial_experience_id}')
 178         episode, season, show = self._get_episode(webpage, experience_id=int(initial_experience_id))
 179         episode_id = str(episode['episodePk'])
 180         display_id = episode.get('slug') or episode_id
 181
 182         formats, subtitles, thumbnails, duration = [], {}, [], 0
 183         requested_languages, requested_versions = self._configuration_arg('language'), self._configuration_arg('version')
 184         language_preference = qualities((requested_languages or [''])[::-1])
 185         source_preference = qualities((requested_versions or ['uncut', 'simulcast'])[::-1])
 186         only_initial_experience = 'seperate-video-versions' in self.get_param('compat_opts', [])
 187
 188         for lang, version, fmt in self._get_experiences(episode):
 189             experience_id = str(fmt['experienceId'])
 190             if (only_initial_experience and experience_id != initial_experience_id
 191                     or requested_languages and lang.lower() not in requested_languages
 192                     or requested_versions and version.lower() not in requested_versions):
 193                 continue
 194             thumbnails.append({'url': fmt.get('poster')})
 195             duration = max(duration, fmt.get('duration', 0))
 196             format_name = '%s %s (%s)' % (version, lang, experience_id)
 197             self.extract_subtitles(
 198                 subtitles, experience_id, display_id=display_id, format_name=format_name,
 199                 episode=episode if experience_id == initial_experience_id else episode_id)
 200
 201             headers = {}
 202             if self._TOKEN:
 203                 headers['Authorization'] = 'Token %s' % self._TOKEN
 204             page = self._download_json(
 205                 'https://www.funimation.com/api/showexperience/%s/' % experience_id,
 206                 display_id, headers=headers, expected_status=403, query={
 207                     'pinst_id': ''.join([random.choice(string.digits + string.ascii_letters) for _ in range(8)]),
 208                 }, note=f'Downloading {format_name} JSON')
 209             sources = page.get('items') or []
 210             if not sources:
 211                 error = try_get(page, lambda x: x['errors'][0], dict)
 212                 if error:
 213                     self.report_warning('%s said: Error %s - %s' % (
 214                         self.IE_NAME, error.get('code'), error.get('detail') or error.get('title')))
 215                 else:
 216                     self.report_warning('No sources found for format')
 217
 218             current_formats = []
 219             for source in sources:
 220                 source_url = source.get('src')
 221                 source_type = source.get('videoType') or determine_ext(source_url)
 222                 if source_type == 'm3u8':
 223                     current_formats.extend(self._extract_m3u8_formats(
 224                         source_url, display_id, 'mp4', m3u8_id='%s-%s' % (experience_id, 'hls'), fatal=False,
 225                         note=f'Downloading {format_name} m3u8 information'))
 226                 else:
 227                     current_formats.append({
 228                         'format_id': '%s-%s' % (experience_id, source_type),
 229                         'url': source_url,
 230                     })
 231                 for f in current_formats:
 232                     # TODO: Convert language to code
 233                     f.update({
 234                         'language': lang,
 235                         'format_note': version,
 236                         'source_preference': source_preference(version.lower()),
 237                         'language_preference': language_preference(lang.lower()),
 238                     })
 239                 formats.extend(current_formats)
 240         self._remove_duplicate_formats(formats)
 241         self._sort_formats(formats, ('lang', 'source'))
 242
 243         return {
 244             'id': initial_experience_id if only_initial_experience else episode_id,
 245             'display_id': display_id,
 246             'duration': duration,
 247             'title': episode['episodeTitle'],
 248             'description': episode.get('episodeSummary'),
 249             'episode': episode.get('episodeTitle'),
 250             'episode_number': int_or_none(episode.get('episodeId')),
 251             'episode_id': episode_id,
 252             'season': season.get('seasonTitle'),
 253             'season_number': int_or_none(season.get('seasonId')),
 254             'season_id': str_or_none(season.get('seasonPk')),
 255             'series': show.get('showTitle'),
 256             'formats': formats,
 257             'thumbnails': thumbnails,
 258             'subtitles': subtitles,
 259         }
 260
 261     def _get_subtitles(self, subtitles, experience_id, episode, display_id, format_name):
 262         if isinstance(episode, str):
 263             webpage = self._download_webpage(
 264                 f'https://www.funimation.com/player/{experience_id}', display_id,
 265                 fatal=False, note=f'Downloading player webpage for {format_name}')
 266             episode, _, _ = self._get_episode(webpage, episode_id=episode, fatal=False)
 267
 268         for _, version, f in self._get_experiences(episode):
 269             for source in f.get('sources'):
 270                 for text_track in source.get('textTracks'):
 271                     if not text_track.get('src'):
 272                         continue
 273                     sub_type = text_track.get('type').upper()
 274                     sub_type = sub_type if sub_type != 'FULL' else None
 275                     current_sub = {
 276                         'url': text_track['src'],
 277                         'name': ' '.join(filter(None, (version, text_track.get('label'), sub_type)))
 278                     }
 279                     lang = '_'.join(filter(None, (
 280                         text_track.get('language', 'und'), version if version != 'Simulcast' else None, sub_type)))
 281                     if current_sub not in subtitles.get(lang, []):
 282                         subtitles.setdefault(lang, []).append(current_sub)
 283         return subtitles
 284
 285
 286 class FunimationShowIE(FunimationIE):
 287     IE_NAME = 'funimation:show'
 288     _VALID_URL = r'(?P<url>https?://(?:www\.)?funimation(?:\.com|now\.uk)/(?P<locale>[^/]+)?/?shows/(?P<id>[^/?#&]+))/?(?:[?#]|$)'
 289
 290     _TESTS = [{
 291         'url': 'https://www.funimation.com/en/shows/sk8-the-infinity',
 292         'info_dict': {
 293             'id': 1315000,
 294             'title': 'SK8 the Infinity'
 295         },
 296         'playlist_count': 13,
 297         'params': {
 298             'skip_download': True,
 299         },
 300     }, {
 301         # without lang code
 302         'url': 'https://www.funimation.com/shows/ouran-high-school-host-club/',
 303         'info_dict': {
 304             'id': 39643,
 305             'title': 'Ouran High School Host Club'
 306         },
 307         'playlist_count': 26,
 308         'params': {
 309             'skip_download': True,
 310         },
 311     }]
 312
 313     def _real_initialize(self):
 314         region = self._get_cookies('https://www.funimation.com').get('region')
 315         self._region = region.value if region else try_get(
 316             self._download_json(
 317                 'https://geo-service.prd.funimationsvc.com/geo/v1/region/check', None, fatal=False,
 318                 note='Checking geo-location', errnote='Unable to fetch geo-location information'),
 319             lambda x: x['region']) or 'US'
 320
 321     def _real_extract(self, url):
 322         base_url, locale, display_id = self._match_valid_url(url).groups()
 323
 324         show_info = self._download_json(
 325             'https://title-api.prd.funimationsvc.com/v2/shows/%s?region=%s&deviceType=web&locale=%s'
 326             % (display_id, self._region, locale or 'en'), display_id)
 327         items = self._download_json(
 328             'https://prod-api-funimationnow.dadcdigital.com/api/funimation/episodes/?limit=99999&title_id=%s'
 329             % show_info.get('id'), display_id).get('items')
 330         vod_items = map(lambda k: dict_get(k, ('mostRecentSvod', 'mostRecentAvod')).get('item'), items)
 331
 332         return {
 333             '_type': 'playlist',
 334             'id': show_info['id'],
 335             'title': show_info['name'],
 336             'entries': [
 337                 self.url_result(
 338                     '%s/%s' % (base_url, vod_item.get('episodeSlug')), FunimationPageIE.ie_key(),
 339                     vod_item.get('episodeId'), vod_item.get('episodeName'))
 340                 for vod_item in sorted(vod_items, key=lambda x: x.get('episodeOrder'))],
 341         }