dict_get,
int_or_none,
js_to_json,
+ str_or_none,
+ try_get,
urlencode_postdata,
- urljoin,
ExtractorError,
)
+class FunimationPageIE(InfoExtractor):
+ IE_NAME = 'funimation:page'
+ _VALID_URL = r'(?P<origin>https?://(?:www\.)?funimation(?:\.com|now\.uk))/(?P<lang>[^/]+/)?(?P<path>shows/(?P<id>[^/]+/[^/?#&]+).*$)'
+
+ _TESTS = [{
+ 'url': 'https://www.funimation.com/shows/attack-on-titan-junior-high/broadcast-dub-preview/',
+ 'info_dict': {
+ 'id': '210050',
+ 'ext': 'mp4',
+ 'title': 'Broadcast Dub Preview',
+ # Other metadata is tested in FunimationIE
+ },
+ 'params': {
+ 'skip_download': 'm3u8',
+ },
+ 'add_ie': ['Funimation'],
+ }, {
+ # Not available in US
+ 'url': 'https://www.funimation.com/shows/hacksign/role-play/',
+ 'only_matching': True,
+ }, {
+ # with lang code
+ 'url': 'https://www.funimation.com/en/shows/hacksign/role-play/',
+ 'only_matching': True,
+ }, {
+ 'url': 'https://www.funimationnow.uk/shows/puzzle-dragons-x/drop-impact/simulcast/',
+ 'only_matching': True,
+ }]
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ display_id = mobj.group('id').replace('/', '_')
+ if not mobj.group('lang'):
+ url = '%s/en/%s' % (mobj.group('origin'), mobj.group('path'))
+
+ webpage = self._download_webpage(url, display_id)
+ title_data = self._parse_json(self._search_regex(
+ r'TITLE_DATA\s*=\s*({[^}]+})',
+ webpage, 'title data', default=''),
+ display_id, js_to_json, fatal=False) or {}
+
+ video_id = (
+ title_data.get('id')
+ or self._search_regex(
+ (r"KANE_customdimensions.videoID\s*=\s*'(\d+)';", r'<iframe[^>]+src="/player/(\d+)'),
+ webpage, 'video_id', default=None)
+ or self._search_regex(
+ r'/player/(\d+)',
+ self._html_search_meta(['al:web:url', 'og:video:url', 'og:video:secure_url'], webpage, fatal=True),
+ 'video id'))
+ return self.url_result(f'https://www.funimation.com/player/{video_id}', FunimationIE.ie_key(), video_id)
+
+
class FunimationIE(InfoExtractor):
- _VALID_URL = r'https?://(?:www\.)?funimation(?:\.com|now\.uk)/(?:[^/]+/)?shows/[^/]+/(?P<id>[^/?#&]+)'
+ _VALID_URL = r'https?://(?:www\.)?funimation\.com/player/(?P<id>\d+)'
_NETRC_MACHINE = 'funimation'
_TOKEN = None
_TESTS = [{
- 'url': 'https://www.funimation.com/shows/hacksign/role-play/',
+ 'url': 'https://www.funimation.com/player/210051',
'info_dict': {
- 'id': '91144',
- 'display_id': 'role-play',
+ 'id': '210050',
+ 'display_id': 'broadcast-dub-preview',
'ext': 'mp4',
- 'title': '.hack//SIGN - Role Play',
- 'description': 'md5:b602bdc15eef4c9bbb201bb6e6a4a2dd',
- 'thumbnail': r're:https?://.*\.jpg',
+ 'title': 'Broadcast Dub Preview',
+ 'thumbnail': r're:https?://.*\.(?:jpg|png)',
+ 'episode': 'Broadcast Dub Preview',
+ 'episode_id': '210050',
+ 'season': 'Extras',
+ 'season_id': '166038',
+ 'season_number': 99,
+ 'series': 'Attack on Titan: Junior High',
+ 'description': '',
+ 'duration': 154,
},
'params': {
- # m3u8 download
- 'skip_download': True,
+ 'skip_download': 'm3u8',
},
}, {
- 'url': 'https://www.funimation.com/shows/attack-on-titan-junior-high/broadcast-dub-preview/',
+ 'note': 'player_id should be extracted with the relevent compat-opt',
+ 'url': 'https://www.funimation.com/player/210051',
'info_dict': {
'id': '210051',
'display_id': 'broadcast-dub-preview',
'ext': 'mp4',
- 'title': 'Attack on Titan: Junior High - Broadcast Dub Preview',
+ 'title': 'Broadcast Dub Preview',
'thumbnail': r're:https?://.*\.(?:jpg|png)',
+ 'episode': 'Broadcast Dub Preview',
+ 'episode_id': '210050',
+ 'season': 'Extras',
+ 'season_id': '166038',
+ 'season_number': 99,
+ 'series': 'Attack on Titan: Junior High',
+ 'description': '',
+ 'duration': 154,
},
'params': {
- # m3u8 download
- 'skip_download': True,
+ 'skip_download': 'm3u8',
+ 'compat_opts': ['seperate-video-versions'],
},
- }, {
- 'url': 'https://www.funimationnow.uk/shows/puzzle-dragons-x/drop-impact/simulcast/',
- 'only_matching': True,
- }, {
- # with lang code
- 'url': 'https://www.funimation.com/en/shows/hacksign/role-play/',
- 'only_matching': True,
}]
def _login(self):
def _real_initialize(self):
self._login()
- def _real_extract(self, url):
- display_id = self._match_id(url)
- webpage = self._download_webpage(url, display_id)
+ @staticmethod
+ def _get_experiences(episode):
+ for lang, lang_data in episode.get('languages', {}).items():
+ for video_data in lang_data.values():
+ for version, f in video_data.items():
+ yield lang, version.title(), f
- def _search_kane(name):
- return self._search_regex(
- r"KANE_customdimensions\.%s\s*=\s*'([^']+)';" % name,
- webpage, name, default=None)
+ def _get_episode(self, webpage, experience_id=None, episode_id=None, fatal=True):
+ ''' Extract the episode, season and show objects given either episode/experience id '''
+ show = self._parse_json(
+ self._search_regex(
+ r'show\s*=\s*({.+?})\s*;', webpage, 'show data', fatal=fatal),
+ experience_id, transform_source=js_to_json, fatal=fatal) or []
+ for season in show.get('seasons', []):
+ for episode in season.get('episodes', []):
+ if episode_id is not None:
+ if str(episode.get('episodePk')) == episode_id:
+ return episode, season, show
+ continue
+ for _, _, f in self._get_experiences(episode):
+ if f.get('experienceId') == experience_id:
+ return episode, season, show
+ if fatal:
+ raise ExtractorError('Unable to find episode information')
+ else:
+ self.report_warning('Unable to find episode information')
+ return {}, {}, {}
- title_data = self._parse_json(self._search_regex(
- r'TITLE_DATA\s*=\s*({[^}]+})',
- webpage, 'title data', default=''),
- display_id, js_to_json, fatal=False) or {}
+ def _real_extract(self, url):
+ initial_experience_id = self._match_id(url)
+ webpage = self._download_webpage(
+ url, initial_experience_id, note=f'Downloading player webpage for {initial_experience_id}')
+ episode, season, show = self._get_episode(webpage, experience_id=int(initial_experience_id))
+ episode_id = str(episode['episodePk'])
+ display_id = episode.get('slug') or episode_id
- video_id = title_data.get('id') or self._search_regex([
- r"KANE_customdimensions.videoID\s*=\s*'(\d+)';",
- r'<iframe[^>]+src="/player/(\d+)',
- ], webpage, 'video_id', default=None)
- if not video_id:
- player_url = self._html_search_meta([
- 'al:web:url',
- 'og:video:url',
- 'og:video:secure_url',
- ], webpage, fatal=True)
- video_id = self._search_regex(r'/player/(\d+)', player_url, 'video id')
-
- title = episode = title_data.get('title') or _search_kane('videoTitle') or self._og_search_title(webpage)
- series = _search_kane('showName')
- if series:
- title = '%s - %s' % (series, title)
- description = self._html_search_meta(['description', 'og:description'], webpage, fatal=True)
- subtitles = self.extract_subtitles(url, video_id, display_id)
+ formats, subtitles, thumbnails, duration = [], {}, [], 0
+ requested_languages, requested_versions = self._configuration_arg('language'), self._configuration_arg('version')
+ only_initial_experience = 'seperate-video-versions' in self.get_param('compat_opts', [])
+
+ for lang, version, fmt in self._get_experiences(episode):
+ experience_id = str(fmt['experienceId'])
+ if (only_initial_experience and experience_id != initial_experience_id
+ or requested_languages and lang not in requested_languages
+ or requested_versions and version not in requested_versions):
+ continue
+ thumbnails.append({'url': fmt.get('poster')})
+ duration = max(duration, fmt.get('duration', 0))
+ format_name = '%s %s (%s)' % (version, lang, experience_id)
+ self.extract_subtitles(
+ subtitles, experience_id, display_id=display_id, format_name=format_name,
+ episode=episode if experience_id == initial_experience_id else episode_id)
- try:
headers = {}
if self._TOKEN:
headers['Authorization'] = 'Token %s' % self._TOKEN
- sources = self._download_json(
- 'https://www.funimation.com/api/showexperience/%s/' % video_id,
- video_id, headers=headers, query={
+ page = self._download_json(
+ 'https://www.funimation.com/api/showexperience/%s/' % experience_id,
+ display_id, headers=headers, expected_status=403, query={
'pinst_id': ''.join([random.choice(string.digits + string.ascii_letters) for _ in range(8)]),
- })['items']
- except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
- error = self._parse_json(e.cause.read(), video_id)['errors'][0]
- raise ExtractorError('%s said: %s' % (
- self.IE_NAME, error.get('detail') or error.get('title')), expected=True)
- raise
+ }, note=f'Downloading {format_name} JSON')
+ sources = page.get('items') or []
+ if not sources:
+ error = try_get(page, lambda x: x['errors'][0], dict)
+ if error:
+ self.report_warning('%s said: Error %s - %s' % (
+ self.IE_NAME, error.get('code'), error.get('detail') or error.get('title')))
+ else:
+ self.report_warning('No sources found for format')
- formats = []
- for source in sources:
- source_url = source.get('src')
- if not source_url:
- continue
- source_type = source.get('videoType') or determine_ext(source_url)
- if source_type == 'm3u8':
- formats.extend(self._extract_m3u8_formats(
- source_url, video_id, 'mp4',
- m3u8_id='hls', fatal=False))
- else:
- formats.append({
- 'format_id': source_type,
- 'url': source_url,
- })
+ current_formats = []
+ for source in sources:
+ source_url = source.get('src')
+ source_type = source.get('videoType') or determine_ext(source_url)
+ if source_type == 'm3u8':
+ current_formats.extend(self._extract_m3u8_formats(
+ source_url, display_id, 'mp4', m3u8_id='%s-%s' % (experience_id, 'hls'), fatal=False,
+ note=f'Downloading {format_name} m3u8 information'))
+ else:
+ current_formats.append({
+ 'format_id': '%s-%s' % (experience_id, source_type),
+ 'url': source_url,
+ })
+ for f in current_formats:
+ # TODO: Convert language to code
+ f.update({'language': lang, 'format_note': version})
+ formats.extend(current_formats)
+ self._remove_duplicate_formats(formats)
self._sort_formats(formats)
return {
- 'id': video_id,
+ 'id': initial_experience_id if only_initial_experience else episode_id,
'display_id': display_id,
- 'title': title,
- 'description': description,
- 'thumbnail': self._og_search_thumbnail(webpage),
- 'series': series,
- 'season_number': int_or_none(title_data.get('seasonNum') or _search_kane('season')),
- 'episode_number': int_or_none(title_data.get('episodeNum')),
- 'episode': episode,
- 'subtitles': subtitles,
- 'season_id': title_data.get('seriesId'),
+ 'duration': duration,
+ 'title': episode['episodeTitle'],
+ 'description': episode.get('episodeSummary'),
+ 'episode': episode.get('episodeTitle'),
+ 'episode_number': int_or_none(episode.get('episodeId')),
+ 'episode_id': episode_id,
+ 'season': season.get('seasonTitle'),
+ 'season_number': int_or_none(season.get('seasonId')),
+ 'season_id': str_or_none(season.get('seasonPk')),
+ 'series': show.get('showTitle'),
'formats': formats,
+ 'thumbnails': thumbnails,
+ 'subtitles': subtitles,
}
- def _get_subtitles(self, url, video_id, display_id):
- player_url = urljoin(url, '/player/' + video_id)
- player_page = self._download_webpage(player_url, display_id)
- text_tracks_json_string = self._search_regex(
- r'"textTracks": (\[{.+?}\])',
- player_page, 'subtitles data', default='')
- text_tracks = self._parse_json(
- text_tracks_json_string, display_id, js_to_json, fatal=False) or []
- subtitles = {}
- for text_track in text_tracks:
- url_element = {'url': text_track.get('src')}
- language = text_track.get('language')
- if text_track.get('type') == 'CC':
- language += '_CC'
- subtitles.setdefault(language, []).append(url_element)
+ def _get_subtitles(self, subtitles, experience_id, episode, display_id, format_name):
+ if isinstance(episode, str):
+ webpage = self._download_webpage(
+ f'https://www.funimation.com/player/{experience_id}', display_id,
+ fatal=False, note=f'Downloading player webpage for {format_name}')
+ episode, _, _ = self._get_episode(webpage, episode_id=episode, fatal=False)
+
+ for _, version, f in self._get_experiences(episode):
+ for source in f.get('sources'):
+ for text_track in source.get('textTracks'):
+ if not text_track.get('src'):
+ continue
+ sub_type = text_track.get('type').upper()
+ sub_type = sub_type if sub_type != 'FULL' else None
+ current_sub = {
+ 'url': text_track['src'],
+ 'name': ' '.join(filter(None, (version, text_track.get('label'), sub_type)))
+ }
+ lang = '_'.join(filter(None, (
+ text_track.get('language', 'und'), version if version != 'Simulcast' else None, sub_type)))
+ if current_sub not in subtitles.get(lang, []):
+ subtitles.setdefault(lang, []).append(current_sub)
return subtitles
'title': show_info['name'],
'entries': [
self.url_result(
- '%s/%s' % (base_url, vod_item.get('episodeSlug')), FunimationIE.ie_key(),
+ '%s/%s' % (base_url, vod_item.get('episodeSlug')), FunimationPageIE.ie_key(),
vod_item.get('episodeId'), vod_item.get('episodeName'))
for vod_item in sorted(vod_items, key=lambda x: x.get('episodeOrder'))],
}
projects / yt-dlp.git / commitdiff
