]>
Commit | Line | Data |
---|---|---|
929ba399 | 1 | import random |
8cd69fc4 | 2 | import re |
929ba399 RA |
3 | import string |
4 | ||
ab4bdc91 | 5 | from .common import InfoExtractor |
804181dd | 6 | from ..compat import compat_HTTPError |
ab4bdc91 | 7 | from ..utils import ( |
0647d925 | 8 | ExtractorError, |
f542a3d2 | 9 | determine_ext, |
f377f44d | 10 | int_or_none, |
34921b43 | 11 | join_nonempty, |
91399b2f | 12 | js_to_json, |
0647d925 | 13 | make_archive_id, |
8cd69fc4 J |
14 | orderedSet, |
15 | qualities, | |
3acf6d38 | 16 | str_or_none, |
8cd69fc4 | 17 | traverse_obj, |
3acf6d38 | 18 | try_get, |
29f63c96 | 19 | urlencode_postdata, |
ab4bdc91 | 20 | ) |
ab4bdc91 | 21 | |
b4c299ba | 22 | |
8cd69fc4 J |
23 | class FunimationBaseIE(InfoExtractor): |
24 | _NETRC_MACHINE = 'funimation' | |
25 | _REGION = None | |
26 | _TOKEN = None | |
27 | ||
28 | def _get_region(self): | |
29 | region_cookie = self._get_cookies('https://www.funimation.com').get('region') | |
30 | region = region_cookie.value if region_cookie else self.get_param('geo_bypass_country') | |
31 | return region or traverse_obj( | |
32 | self._download_json( | |
33 | 'https://geo-service.prd.funimationsvc.com/geo/v1/region/check', None, fatal=False, | |
34 | note='Checking geo-location', errnote='Unable to fetch geo-location information'), | |
35 | 'region') or 'US' | |
36 | ||
52efa4b3 | 37 | def _perform_login(self, username, password): |
38 | if self._TOKEN: | |
8cd69fc4 J |
39 | return |
40 | try: | |
41 | data = self._download_json( | |
42 | 'https://prod-api-funimationnow.dadcdigital.com/api/auth/login/', | |
43 | None, 'Logging in', data=urlencode_postdata({ | |
44 | 'username': username, | |
45 | 'password': password, | |
46 | })) | |
52efa4b3 | 47 | FunimationBaseIE._TOKEN = data['token'] |
8cd69fc4 J |
48 | except ExtractorError as e: |
49 | if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401: | |
50 | error = self._parse_json(e.cause.read().decode(), None)['error'] | |
51 | raise ExtractorError(error, expected=True) | |
52 | raise | |
53 | ||
54 | ||
55 | class FunimationPageIE(FunimationBaseIE): | |
3acf6d38 | 56 | IE_NAME = 'funimation:page' |
8cd69fc4 | 57 | _VALID_URL = r'https?://(?:www\.)?funimation(?:\.com|now\.uk)/(?:(?P<lang>[^/]+)/)?(?:shows|v)/(?P<show>[^/]+)/(?P<episode>[^/?#&]+)' |
3acf6d38 | 58 | |
59 | _TESTS = [{ | |
60 | 'url': 'https://www.funimation.com/shows/attack-on-titan-junior-high/broadcast-dub-preview/', | |
61 | 'info_dict': { | |
62 | 'id': '210050', | |
63 | 'ext': 'mp4', | |
64 | 'title': 'Broadcast Dub Preview', | |
65 | # Other metadata is tested in FunimationIE | |
66 | }, | |
67 | 'params': { | |
68 | 'skip_download': 'm3u8', | |
69 | }, | |
70 | 'add_ie': ['Funimation'], | |
71 | }, { | |
72 | # Not available in US | |
73 | 'url': 'https://www.funimation.com/shows/hacksign/role-play/', | |
74 | 'only_matching': True, | |
75 | }, { | |
76 | # with lang code | |
77 | 'url': 'https://www.funimation.com/en/shows/hacksign/role-play/', | |
78 | 'only_matching': True, | |
79 | }, { | |
80 | 'url': 'https://www.funimationnow.uk/shows/puzzle-dragons-x/drop-impact/simulcast/', | |
81 | 'only_matching': True, | |
8cd69fc4 J |
82 | }, { |
83 | 'url': 'https://www.funimation.com/v/a-certain-scientific-railgun/super-powered-level-5', | |
84 | 'only_matching': True, | |
3acf6d38 | 85 | }] |
86 | ||
8cd69fc4 J |
87 | def _real_initialize(self): |
88 | if not self._REGION: | |
89 | FunimationBaseIE._REGION = self._get_region() | |
8cd69fc4 | 90 | |
3acf6d38 | 91 | def _real_extract(self, url): |
8cd69fc4 J |
92 | locale, show, episode = self._match_valid_url(url).group('lang', 'show', 'episode') |
93 | ||
94 | video_id = traverse_obj(self._download_json( | |
95 | f'https://title-api.prd.funimationsvc.com/v1/shows/{show}/episodes/{episode}', | |
96 | f'{show}_{episode}', query={ | |
97 | 'deviceType': 'web', | |
98 | 'region': self._REGION, | |
99 | 'locale': locale or 'en' | |
100 | }), ('videoList', ..., 'id'), get_all=False) | |
101 | ||
3acf6d38 | 102 | return self.url_result(f'https://www.funimation.com/player/{video_id}', FunimationIE.ie_key(), video_id) |
103 | ||
104 | ||
8cd69fc4 | 105 | class FunimationIE(FunimationBaseIE): |
3acf6d38 | 106 | _VALID_URL = r'https?://(?:www\.)?funimation\.com/player/(?P<id>\d+)' |
ab4bdc91 | 107 | |
b59623ef | 108 | _TESTS = [{ |
3acf6d38 | 109 | 'url': 'https://www.funimation.com/player/210051', |
ab4bdc91 | 110 | 'info_dict': { |
3acf6d38 | 111 | 'id': '210050', |
112 | 'display_id': 'broadcast-dub-preview', | |
b59623ef | 113 | 'ext': 'mp4', |
3acf6d38 | 114 | 'title': 'Broadcast Dub Preview', |
115 | 'thumbnail': r're:https?://.*\.(?:jpg|png)', | |
116 | 'episode': 'Broadcast Dub Preview', | |
117 | 'episode_id': '210050', | |
118 | 'season': 'Extras', | |
119 | 'season_id': '166038', | |
120 | 'season_number': 99, | |
121 | 'series': 'Attack on Titan: Junior High', | |
122 | 'description': '', | |
8cd69fc4 | 123 | 'duration': 155, |
b59623ef | 124 | }, |
91399b2f | 125 | 'params': { |
3acf6d38 | 126 | 'skip_download': 'm3u8', |
91399b2f | 127 | }, |
b091529a | 128 | }, { |
3acf6d38 | 129 | 'note': 'player_id should be extracted with the relevent compat-opt', |
130 | 'url': 'https://www.funimation.com/player/210051', | |
0b1bb1ac | 131 | 'info_dict': { |
804181dd | 132 | 'id': '210051', |
0b1bb1ac S |
133 | 'display_id': 'broadcast-dub-preview', |
134 | 'ext': 'mp4', | |
3acf6d38 | 135 | 'title': 'Broadcast Dub Preview', |
ec85ded8 | 136 | 'thumbnail': r're:https?://.*\.(?:jpg|png)', |
3acf6d38 | 137 | 'episode': 'Broadcast Dub Preview', |
138 | 'episode_id': '210050', | |
139 | 'season': 'Extras', | |
140 | 'season_id': '166038', | |
141 | 'season_number': 99, | |
142 | 'series': 'Attack on Titan: Junior High', | |
143 | 'description': '', | |
8cd69fc4 | 144 | 'duration': 155, |
0b1bb1ac | 145 | }, |
804181dd | 146 | 'params': { |
3acf6d38 | 147 | 'skip_download': 'm3u8', |
148 | 'compat_opts': ['seperate-video-versions'], | |
804181dd | 149 | }, |
b59623ef | 150 | }] |
f542a3d2 | 151 | |
3acf6d38 | 152 | @staticmethod |
153 | def _get_experiences(episode): | |
154 | for lang, lang_data in episode.get('languages', {}).items(): | |
155 | for video_data in lang_data.values(): | |
156 | for version, f in video_data.items(): | |
157 | yield lang, version.title(), f | |
f542a3d2 | 158 | |
3acf6d38 | 159 | def _get_episode(self, webpage, experience_id=None, episode_id=None, fatal=True): |
160 | ''' Extract the episode, season and show objects given either episode/experience id ''' | |
161 | show = self._parse_json( | |
162 | self._search_regex( | |
163 | r'show\s*=\s*({.+?})\s*;', webpage, 'show data', fatal=fatal), | |
164 | experience_id, transform_source=js_to_json, fatal=fatal) or [] | |
165 | for season in show.get('seasons', []): | |
166 | for episode in season.get('episodes', []): | |
167 | if episode_id is not None: | |
168 | if str(episode.get('episodePk')) == episode_id: | |
169 | return episode, season, show | |
170 | continue | |
171 | for _, _, f in self._get_experiences(episode): | |
172 | if f.get('experienceId') == experience_id: | |
173 | return episode, season, show | |
174 | if fatal: | |
175 | raise ExtractorError('Unable to find episode information') | |
176 | else: | |
177 | self.report_warning('Unable to find episode information') | |
178 | return {}, {}, {} | |
91399b2f | 179 | |
3acf6d38 | 180 | def _real_extract(self, url): |
181 | initial_experience_id = self._match_id(url) | |
182 | webpage = self._download_webpage( | |
183 | url, initial_experience_id, note=f'Downloading player webpage for {initial_experience_id}') | |
184 | episode, season, show = self._get_episode(webpage, experience_id=int(initial_experience_id)) | |
185 | episode_id = str(episode['episodePk']) | |
186 | display_id = episode.get('slug') or episode_id | |
91399b2f | 187 | |
3acf6d38 | 188 | formats, subtitles, thumbnails, duration = [], {}, [], 0 |
189 | requested_languages, requested_versions = self._configuration_arg('language'), self._configuration_arg('version') | |
e919569e | 190 | language_preference = qualities((requested_languages or [''])[::-1]) |
191 | source_preference = qualities((requested_versions or ['uncut', 'simulcast'])[::-1]) | |
3acf6d38 | 192 | only_initial_experience = 'seperate-video-versions' in self.get_param('compat_opts', []) |
193 | ||
194 | for lang, version, fmt in self._get_experiences(episode): | |
195 | experience_id = str(fmt['experienceId']) | |
196 | if (only_initial_experience and experience_id != initial_experience_id | |
4bb6b02f | 197 | or requested_languages and lang.lower() not in requested_languages |
198 | or requested_versions and version.lower() not in requested_versions): | |
3acf6d38 | 199 | continue |
200 | thumbnails.append({'url': fmt.get('poster')}) | |
201 | duration = max(duration, fmt.get('duration', 0)) | |
202 | format_name = '%s %s (%s)' % (version, lang, experience_id) | |
203 | self.extract_subtitles( | |
204 | subtitles, experience_id, display_id=display_id, format_name=format_name, | |
205 | episode=episode if experience_id == initial_experience_id else episode_id) | |
f542a3d2 | 206 | |
8fa17117 RA |
207 | headers = {} |
208 | if self._TOKEN: | |
209 | headers['Authorization'] = 'Token %s' % self._TOKEN | |
3acf6d38 | 210 | page = self._download_json( |
211 | 'https://www.funimation.com/api/showexperience/%s/' % experience_id, | |
212 | display_id, headers=headers, expected_status=403, query={ | |
efa944f4 | 213 | 'pinst_id': ''.join(random.choices(string.digits + string.ascii_letters, k=8)), |
3acf6d38 | 214 | }, note=f'Downloading {format_name} JSON') |
215 | sources = page.get('items') or [] | |
216 | if not sources: | |
217 | error = try_get(page, lambda x: x['errors'][0], dict) | |
218 | if error: | |
219 | self.report_warning('%s said: Error %s - %s' % ( | |
220 | self.IE_NAME, error.get('code'), error.get('detail') or error.get('title'))) | |
221 | else: | |
222 | self.report_warning('No sources found for format') | |
f542a3d2 | 223 | |
3acf6d38 | 224 | current_formats = [] |
225 | for source in sources: | |
226 | source_url = source.get('src') | |
227 | source_type = source.get('videoType') or determine_ext(source_url) | |
228 | if source_type == 'm3u8': | |
229 | current_formats.extend(self._extract_m3u8_formats( | |
230 | source_url, display_id, 'mp4', m3u8_id='%s-%s' % (experience_id, 'hls'), fatal=False, | |
231 | note=f'Downloading {format_name} m3u8 information')) | |
232 | else: | |
233 | current_formats.append({ | |
234 | 'format_id': '%s-%s' % (experience_id, source_type), | |
235 | 'url': source_url, | |
236 | }) | |
237 | for f in current_formats: | |
238 | # TODO: Convert language to code | |
e919569e | 239 | f.update({ |
240 | 'language': lang, | |
241 | 'format_note': version, | |
242 | 'source_preference': source_preference(version.lower()), | |
243 | 'language_preference': language_preference(lang.lower()), | |
244 | }) | |
3acf6d38 | 245 | formats.extend(current_formats) |
1d485a1a | 246 | if not formats and (requested_languages or requested_versions): |
247 | self.raise_no_formats( | |
248 | 'There are no video formats matching the requested languages/versions', expected=True, video_id=display_id) | |
3acf6d38 | 249 | self._remove_duplicate_formats(formats) |
b59623ef | 250 | |
ab4bdc91 | 251 | return { |
1e8fe57e | 252 | 'id': episode_id, |
0647d925 | 253 | '_old_archive_ids': [make_archive_id(self, initial_experience_id)], |
f542a3d2 | 254 | 'display_id': display_id, |
3acf6d38 | 255 | 'duration': duration, |
256 | 'title': episode['episodeTitle'], | |
257 | 'description': episode.get('episodeSummary'), | |
258 | 'episode': episode.get('episodeTitle'), | |
259 | 'episode_number': int_or_none(episode.get('episodeId')), | |
260 | 'episode_id': episode_id, | |
261 | 'season': season.get('seasonTitle'), | |
262 | 'season_number': int_or_none(season.get('seasonId')), | |
263 | 'season_id': str_or_none(season.get('seasonPk')), | |
264 | 'series': show.get('showTitle'), | |
ab4bdc91 | 265 | 'formats': formats, |
3acf6d38 | 266 | 'thumbnails': thumbnails, |
267 | 'subtitles': subtitles, | |
9f14daf2 | 268 | '_format_sort_fields': ('lang', 'source'), |
ab4bdc91 | 269 | } |
29f63c96 | 270 | |
3acf6d38 | 271 | def _get_subtitles(self, subtitles, experience_id, episode, display_id, format_name): |
272 | if isinstance(episode, str): | |
273 | webpage = self._download_webpage( | |
9222c381 | 274 | f'https://www.funimation.com/player/{experience_id}/', display_id, |
3acf6d38 | 275 | fatal=False, note=f'Downloading player webpage for {format_name}') |
276 | episode, _, _ = self._get_episode(webpage, episode_id=episode, fatal=False) | |
277 | ||
278 | for _, version, f in self._get_experiences(episode): | |
279 | for source in f.get('sources'): | |
280 | for text_track in source.get('textTracks'): | |
281 | if not text_track.get('src'): | |
282 | continue | |
283 | sub_type = text_track.get('type').upper() | |
284 | sub_type = sub_type if sub_type != 'FULL' else None | |
285 | current_sub = { | |
286 | 'url': text_track['src'], | |
34921b43 | 287 | 'name': join_nonempty(version, text_track.get('label'), sub_type, delim=' ') |
3acf6d38 | 288 | } |
34921b43 | 289 | lang = join_nonempty(text_track.get('language', 'und'), |
290 | version if version != 'Simulcast' else None, | |
291 | sub_type, delim='_') | |
3acf6d38 | 292 | if current_sub not in subtitles.get(lang, []): |
293 | subtitles.setdefault(lang, []).append(current_sub) | |
29f63c96 | 294 | return subtitles |
125728b0 M |
295 | |
296 | ||
8cd69fc4 | 297 | class FunimationShowIE(FunimationBaseIE): |
125728b0 M |
298 | IE_NAME = 'funimation:show' |
299 | _VALID_URL = r'(?P<url>https?://(?:www\.)?funimation(?:\.com|now\.uk)/(?P<locale>[^/]+)?/?shows/(?P<id>[^/?#&]+))/?(?:[?#]|$)' | |
300 | ||
301 | _TESTS = [{ | |
302 | 'url': 'https://www.funimation.com/en/shows/sk8-the-infinity', | |
303 | 'info_dict': { | |
304 | 'id': 1315000, | |
305 | 'title': 'SK8 the Infinity' | |
306 | }, | |
307 | 'playlist_count': 13, | |
308 | 'params': { | |
309 | 'skip_download': True, | |
310 | }, | |
311 | }, { | |
312 | # without lang code | |
313 | 'url': 'https://www.funimation.com/shows/ouran-high-school-host-club/', | |
314 | 'info_dict': { | |
315 | 'id': 39643, | |
316 | 'title': 'Ouran High School Host Club' | |
317 | }, | |
318 | 'playlist_count': 26, | |
319 | 'params': { | |
320 | 'skip_download': True, | |
321 | }, | |
322 | }] | |
323 | ||
ad226b1d | 324 | def _real_initialize(self): |
8cd69fc4 J |
325 | if not self._REGION: |
326 | FunimationBaseIE._REGION = self._get_region() | |
ad226b1d | 327 | |
125728b0 | 328 | def _real_extract(self, url): |
5ad28e7f | 329 | base_url, locale, display_id = self._match_valid_url(url).groups() |
125728b0 M |
330 | |
331 | show_info = self._download_json( | |
ad226b1d | 332 | 'https://title-api.prd.funimationsvc.com/v2/shows/%s?region=%s&deviceType=web&locale=%s' |
8cd69fc4 J |
333 | % (display_id, self._REGION, locale or 'en'), display_id) |
334 | items_info = self._download_json( | |
125728b0 | 335 | 'https://prod-api-funimationnow.dadcdigital.com/api/funimation/episodes/?limit=99999&title_id=%s' |
8cd69fc4 J |
336 | % show_info.get('id'), display_id) |
337 | ||
e6f868a6 | 338 | vod_items = traverse_obj(items_info, ('items', ..., lambda k, _: re.match(r'(?i)mostRecent[AS]vod', k), 'item')) |
125728b0 M |
339 | |
340 | return { | |
341 | '_type': 'playlist', | |
342 | 'id': show_info['id'], | |
343 | 'title': show_info['name'], | |
8cd69fc4 | 344 | 'entries': orderedSet( |
125728b0 | 345 | self.url_result( |
3acf6d38 | 346 | '%s/%s' % (base_url, vod_item.get('episodeSlug')), FunimationPageIE.ie_key(), |
125728b0 | 347 | vod_item.get('episodeId'), vod_item.get('episodeName')) |
8cd69fc4 | 348 | for vod_item in sorted(vod_items, key=lambda x: x.get('episodeOrder', -1))), |
125728b0 | 349 | } |