]>
Commit | Line | Data |
---|---|---|
3121b256 | 1 | from .common import InfoExtractor |
3d2623a8 | 2 | from ..networking.exceptions import HTTPError |
3121b256 NP |
3 | from ..utils import ( |
4 | ExtractorError, | |
dfd8c0b6 | 5 | LazyList, |
865b0872 | 6 | int_or_none, |
244644c0 | 7 | join_nonempty, |
dfd8c0b6 | 8 | parse_iso8601, |
244644c0 | 9 | parse_qs, |
b2cc150a | 10 | smuggle_url, |
c72dc20d | 11 | str_or_none, |
244644c0 | 12 | url_or_none, |
3121b256 | 13 | urlencode_postdata, |
244644c0 | 14 | urljoin, |
3121b256 | 15 | ) |
8993721e | 16 | from ..utils.traversal import traverse_obj |
3121b256 NP |
17 | |
18 | ||
244644c0 | 19 | class RoosterTeethBaseIE(InfoExtractor): |
3121b256 | 20 | _NETRC_MACHINE = 'roosterteeth' |
244644c0 | 21 | _API_BASE = 'https://svod-be.roosterteeth.com' |
22 | _API_BASE_URL = f'{_API_BASE}/api/v1' | |
23 | ||
52efa4b3 | 24 | def _perform_login(self, username, password): |
244644c0 | 25 | if self._get_cookies(self._API_BASE_URL).get('rt_access_token'): |
26 | return | |
27 | ||
28 | try: | |
29 | self._download_json( | |
30 | 'https://auth.roosterteeth.com/oauth/token', | |
31 | None, 'Logging in', data=urlencode_postdata({ | |
32 | 'client_id': '4338d2b4bdc8db1239360f28e72f0d9ddb1fd01e7a38fbb07b4b1f4ba4564cc5', | |
33 | 'grant_type': 'password', | |
34 | 'username': username, | |
35 | 'password': password, | |
36 | })) | |
37 | except ExtractorError as e: | |
38 | msg = 'Unable to login' | |
3d2623a8 | 39 | if isinstance(e.cause, HTTPError) and e.cause.status == 401: |
40 | resp = self._parse_json(e.cause.response.read().decode(), None, fatal=False) | |
244644c0 | 41 | if resp: |
42 | error = resp.get('extra_info') or resp.get('error_description') or resp.get('error') | |
43 | if error: | |
44 | msg += ': ' + error | |
45 | self.report_warning(msg) | |
46 | ||
244644c0 | 47 | def _extract_video_info(self, data): |
48 | thumbnails = [] | |
49 | for image in traverse_obj(data, ('included', 'images')): | |
50 | if image.get('type') not in ('episode_image', 'bonus_feature_image'): | |
51 | continue | |
52 | thumbnails.extend([{ | |
53 | 'id': name, | |
54 | 'url': url, | |
55 | } for name, url in (image.get('attributes') or {}).items() if url_or_none(url)]) | |
56 | ||
57 | attributes = data.get('attributes') or {} | |
58 | title = traverse_obj(attributes, 'title', 'display_title') | |
59 | sub_only = attributes.get('is_sponsors_only') | |
60 | ||
8993721e B |
61 | episode_id = str_or_none(data.get('uuid')) |
62 | video_id = str_or_none(data.get('id')) | |
63 | if video_id and 'parent_content_id' in attributes: # parent_content_id is a bonus-only key | |
64 | video_id += '-bonus' # there are collisions with bonus ids and regular ids | |
65 | elif not video_id: | |
66 | video_id = episode_id | |
67 | ||
244644c0 | 68 | return { |
8993721e | 69 | 'id': video_id, |
244644c0 | 70 | 'display_id': attributes.get('slug'), |
71 | 'title': title, | |
72 | 'description': traverse_obj(attributes, 'description', 'caption'), | |
8993721e | 73 | 'series': traverse_obj(attributes, 'show_title', 'parent_content_title'), |
244644c0 | 74 | 'season_number': int_or_none(attributes.get('season_number')), |
8993721e | 75 | 'season_id': str_or_none(attributes.get('season_id')), |
244644c0 | 76 | 'episode': title, |
77 | 'episode_number': int_or_none(attributes.get('number')), | |
8993721e | 78 | 'episode_id': episode_id, |
244644c0 | 79 | 'channel_id': attributes.get('channel_id'), |
80 | 'duration': int_or_none(attributes.get('length')), | |
dfd8c0b6 | 81 | 'release_timestamp': parse_iso8601(attributes.get('original_air_date')), |
244644c0 | 82 | 'thumbnails': thumbnails, |
83 | 'availability': self._availability( | |
84 | needs_premium=sub_only, needs_subscription=sub_only, needs_auth=sub_only, | |
85 | is_private=False, is_unlisted=False), | |
86 | 'tags': attributes.get('genres') | |
87 | } | |
88 | ||
89 | ||
90 | class RoosterTeethIE(RoosterTeethBaseIE): | |
8993721e | 91 | _VALID_URL = r'https?://(?:.+?\.)?roosterteeth\.com/(?:bonus-feature|episode|watch)/(?P<id>[^/?#&]+)' |
3121b256 NP |
92 | _TESTS = [{ |
93 | 'url': 'http://roosterteeth.com/episode/million-dollars-but-season-2-million-dollars-but-the-game-announcement', | |
94 | 'info_dict': { | |
c72dc20d | 95 | 'id': '9156', |
865b0872 | 96 | 'display_id': 'million-dollars-but-season-2-million-dollars-but-the-game-announcement', |
3121b256 | 97 | 'ext': 'mp4', |
c72dc20d RA |
98 | 'title': 'Million Dollars, But... The Game Announcement', |
99 | 'description': 'md5:168a54b40e228e79f4ddb141e89fe4f5', | |
ec85ded8 | 100 | 'thumbnail': r're:^https?://.*\.png$', |
3121b256 NP |
101 | 'series': 'Million Dollars, But...', |
102 | 'episode': 'Million Dollars, But... The Game Announcement', | |
dd29e6e5 JM |
103 | 'tags': ['Game Show', 'Sketch'], |
104 | 'season_number': 2, | |
105 | 'availability': 'public', | |
106 | 'episode_number': 10, | |
107 | 'episode_id': '00374575-464e-11e7-a302-065410f210c4', | |
108 | 'season': 'Season 2', | |
109 | 'season_id': 'ffa27d48-464d-11e7-a302-065410f210c4', | |
110 | 'channel_id': '92b6bb21-91d2-4b1b-bf95-3268fa0d9939', | |
111 | 'duration': 145, | |
dfd8c0b6 | 112 | 'release_timestamp': 1462982400, |
113 | 'release_date': '20160511', | |
3121b256 | 114 | }, |
b69fd25c | 115 | 'params': {'skip_download': True}, |
b4d10440 FS |
116 | }, { |
117 | 'url': 'https://roosterteeth.com/watch/rwby-bonus-25', | |
b4d10440 | 118 | 'info_dict': { |
244644c0 | 119 | 'id': '40432', |
b4d10440 | 120 | 'display_id': 'rwby-bonus-25', |
244644c0 | 121 | 'title': 'Grimm', |
122 | 'description': 'md5:f30ff570741213418a8d2c19868b93ab', | |
123 | 'episode': 'Grimm', | |
124 | 'channel_id': '92f780eb-ebfe-4bf5-a3b5-c6ad5460a5f1', | |
b4d10440 FS |
125 | 'thumbnail': r're:^https?://.*\.(png|jpe?g)$', |
126 | 'ext': 'mp4', | |
dd29e6e5 JM |
127 | 'availability': 'public', |
128 | 'episode_id': 'f8117b13-f068-499e-803e-eec9ea2dec8c', | |
129 | 'episode_number': 3, | |
130 | 'tags': ['Animation'], | |
131 | 'season_id': '4b8f0a9e-12c4-41ed-8caa-fed15a85bab8', | |
132 | 'season': 'Season 1', | |
133 | 'series': 'RWBY: World of Remnant', | |
134 | 'season_number': 1, | |
135 | 'duration': 216, | |
dfd8c0b6 | 136 | 'release_timestamp': 1413489600, |
137 | 'release_date': '20141016', | |
138 | }, | |
139 | 'params': {'skip_download': True}, | |
8993721e B |
140 | }, { |
141 | # bonus feature with /watch/ url | |
142 | 'url': 'https://roosterteeth.com/watch/rwby-bonus-21', | |
143 | 'info_dict': { | |
144 | 'id': '33-bonus', | |
145 | 'display_id': 'rwby-bonus-21', | |
146 | 'title': 'Volume 5 Yang Character Short', | |
147 | 'description': 'md5:8c2440bc763ea90c52cfe0a68093e1f7', | |
148 | 'episode': 'Volume 5 Yang Character Short', | |
149 | 'channel_id': '92f780eb-ebfe-4bf5-a3b5-c6ad5460a5f1', | |
150 | 'thumbnail': r're:^https?://.*\.(png|jpe?g)$', | |
151 | 'ext': 'mp4', | |
152 | 'availability': 'public', | |
153 | 'episode_id': 'f2a9f132-1fe2-44ad-8956-63d7c0267720', | |
154 | 'episode_number': 55, | |
155 | 'series': 'RWBY', | |
156 | 'duration': 255, | |
157 | 'release_timestamp': 1507993200, | |
158 | 'release_date': '20171014', | |
159 | }, | |
160 | 'params': {'skip_download': True}, | |
dfd8c0b6 | 161 | }, { |
162 | # only works with video_data['attributes']['url'] m3u8 url | |
163 | 'url': 'https://www.roosterteeth.com/watch/achievement-hunter-achievement-hunter-fatality-walkthrough-deathstroke-lex-luthor-captain-marvel-green-lantern-and-wonder-woman', | |
164 | 'info_dict': { | |
165 | 'id': '25394', | |
166 | 'ext': 'mp4', | |
167 | 'title': 'Fatality Walkthrough: Deathstroke, Lex Luthor, Captain Marvel, Green Lantern, and Wonder Woman', | |
168 | 'description': 'md5:91bb934698344fb9647b1c7351f16964', | |
169 | 'availability': 'public', | |
170 | 'thumbnail': r're:^https?://.*\.(png|jpe?g)$', | |
171 | 'episode': 'Fatality Walkthrough: Deathstroke, Lex Luthor, Captain Marvel, Green Lantern, and Wonder Woman', | |
172 | 'episode_number': 71, | |
173 | 'episode_id': 'ffaec998-464d-11e7-a302-065410f210c4', | |
174 | 'season': 'Season 2008', | |
175 | 'tags': ['Gaming'], | |
176 | 'series': 'Achievement Hunter', | |
177 | 'display_id': 'md5:4465ce4f001735f9d7a2ae529a543d31', | |
178 | 'season_id': 'ffa13340-464d-11e7-a302-065410f210c4', | |
179 | 'season_number': 2008, | |
180 | 'channel_id': '2cb2a70c-be50-46f5-93d7-84a1baabb4f7', | |
181 | 'duration': 189, | |
182 | 'release_timestamp': 1228317300, | |
183 | 'release_date': '20081203', | |
b4d10440 | 184 | }, |
b69fd25c | 185 | 'params': {'skip_download': True}, |
b2cc150a | 186 | }, { |
187 | # brightcove fallback extraction needed | |
188 | 'url': 'https://roosterteeth.com/watch/lets-play-2013-126', | |
189 | 'info_dict': { | |
190 | 'id': '17845', | |
191 | 'ext': 'mp4', | |
192 | 'title': 'WWE \'13', | |
193 | 'availability': 'public', | |
194 | 'series': 'Let\'s Play', | |
195 | 'episode_number': 10, | |
196 | 'season_id': 'ffa23d9c-464d-11e7-a302-065410f210c4', | |
197 | 'channel_id': '75ba87e8-06fd-4482-bad9-52a4da2c6181', | |
198 | 'episode': 'WWE \'13', | |
199 | 'episode_id': 'ffdbe55e-464d-11e7-a302-065410f210c4', | |
200 | 'thumbnail': r're:^https?://.*\.(png|jpe?g)$', | |
201 | 'tags': ['Gaming', 'Our Favorites'], | |
202 | 'description': 'md5:b4a5226d2bbcf0dafbde11a2ba27262d', | |
203 | 'display_id': 'lets-play-2013-126', | |
204 | 'season_number': 3, | |
205 | 'season': 'Season 3', | |
206 | 'release_timestamp': 1359999840, | |
207 | 'release_date': '20130204', | |
208 | }, | |
209 | 'expected_warnings': ['Direct m3u8 URL returned HTTP Error 403'], | |
210 | 'params': {'skip_download': True}, | |
3121b256 NP |
211 | }, { |
212 | 'url': 'http://achievementhunter.roosterteeth.com/episode/off-topic-the-achievement-hunter-podcast-2016-i-didn-t-think-it-would-pass-31', | |
213 | 'only_matching': True, | |
214 | }, { | |
215 | 'url': 'http://funhaus.roosterteeth.com/episode/funhaus-shorts-2016-austin-sucks-funhaus-shorts', | |
216 | 'only_matching': True, | |
217 | }, { | |
218 | 'url': 'http://screwattack.roosterteeth.com/episode/death-battle-season-3-mewtwo-vs-shadow', | |
219 | 'only_matching': True, | |
220 | }, { | |
221 | 'url': 'http://theknow.roosterteeth.com/episode/the-know-game-news-season-1-boring-steam-sales-are-better', | |
222 | 'only_matching': True, | |
865b0872 S |
223 | }, { |
224 | # only available for FIRST members | |
225 | 'url': 'http://roosterteeth.com/episode/rt-docs-the-world-s-greatest-head-massage-the-world-s-greatest-head-massage-an-asmr-journey-part-one', | |
226 | 'only_matching': True, | |
5efbc136 RA |
227 | }, { |
228 | 'url': 'https://roosterteeth.com/watch/million-dollars-but-season-2-million-dollars-but-the-game-announcement', | |
229 | 'only_matching': True, | |
8993721e B |
230 | }, { |
231 | 'url': 'https://roosterteeth.com/bonus-feature/camp-camp-soundtrack-another-rap-song-about-foreign-cars-richie-branson', | |
232 | 'only_matching': True, | |
3121b256 | 233 | }] |
3121b256 | 234 | |
b2cc150a | 235 | _BRIGHTCOVE_ACCOUNT_ID = '6203312018001' |
236 | ||
237 | def _extract_brightcove_formats_and_subtitles(self, bc_id, url, m3u8_url): | |
238 | account_id = self._search_regex( | |
239 | r'/accounts/(\d+)/videos/', m3u8_url, 'account id', default=self._BRIGHTCOVE_ACCOUNT_ID) | |
240 | info = self._downloader.get_info_extractor('BrightcoveNew').extract(smuggle_url( | |
241 | f'https://players.brightcove.net/{account_id}/default_default/index.html?videoId={bc_id}', | |
242 | {'referrer': url})) | |
243 | return info['formats'], info['subtitles'] | |
244 | ||
3121b256 | 245 | def _real_extract(self, url): |
865b0872 | 246 | display_id = self._match_id(url) |
244644c0 | 247 | api_episode_url = f'{self._API_BASE_URL}/watch/{display_id}' |
c72dc20d RA |
248 | |
249 | try: | |
dfd14aad | 250 | video_data = self._download_json( |
dd29e6e5 JM |
251 | api_episode_url + '/videos', display_id, 'Downloading video JSON metadata', |
252 | headers={'Client-Type': 'web'})['data'][0] # web client-type yields ad-free streams | |
c72dc20d | 253 | except ExtractorError as e: |
3d2623a8 | 254 | if isinstance(e.cause, HTTPError) and e.cause.status == 403: |
255 | if self._parse_json(e.cause.response.read().decode(), display_id).get('access') is False: | |
c72dc20d RA |
256 | self.raise_login_required( |
257 | '%s is only available for FIRST members' % display_id) | |
258 | raise | |
865b0872 | 259 | |
b2cc150a | 260 | # XXX: additional ad-free URL at video_data['links']['download'] but often gives 403 errors |
261 | m3u8_url = video_data['attributes']['url'] | |
262 | is_brightcove = traverse_obj(video_data, ('attributes', 'encoding_pipeline')) == 'brightcove' | |
263 | bc_id = traverse_obj(video_data, ('attributes', 'uid', {str})) | |
264 | ||
265 | try: | |
266 | formats, subtitles = self._extract_m3u8_formats_and_subtitles( | |
267 | m3u8_url, display_id, 'mp4', 'm3u8_native', m3u8_id='hls') | |
268 | except ExtractorError as e: | |
269 | if is_brightcove and bc_id and isinstance(e.cause, HTTPError) and e.cause.status == 403: | |
270 | self.report_warning( | |
271 | 'Direct m3u8 URL returned HTTP Error 403; retrying with Brightcove extraction') | |
272 | formats, subtitles = self._extract_brightcove_formats_and_subtitles(bc_id, url, m3u8_url) | |
273 | else: | |
274 | raise | |
3121b256 | 275 | |
c72dc20d RA |
276 | episode = self._download_json( |
277 | api_episode_url, display_id, | |
278 | 'Downloading episode JSON metadata')['data'][0] | |
865b0872 | 279 | |
3121b256 | 280 | return { |
865b0872 | 281 | 'display_id': display_id, |
865b0872 | 282 | 'formats': formats, |
244644c0 | 283 | 'subtitles': subtitles, |
284 | **self._extract_video_info(episode) | |
3121b256 | 285 | } |
244644c0 | 286 | |
287 | ||
288 | class RoosterTeethSeriesIE(RoosterTeethBaseIE): | |
289 | _VALID_URL = r'https?://(?:.+?\.)?roosterteeth\.com/series/(?P<id>[^/?#&]+)' | |
290 | _TESTS = [{ | |
291 | 'url': 'https://roosterteeth.com/series/rwby?season=7', | |
292 | 'playlist_count': 13, | |
293 | 'info_dict': { | |
294 | 'id': 'rwby-7', | |
295 | 'title': 'RWBY - Season 7', | |
8993721e B |
296 | }, |
297 | }, { | |
298 | 'url': 'https://roosterteeth.com/series/the-weird-place', | |
299 | 'playlist_count': 7, | |
300 | 'info_dict': { | |
301 | 'id': 'the-weird-place', | |
302 | 'title': 'The Weird Place', | |
303 | }, | |
244644c0 | 304 | }, { |
305 | 'url': 'https://roosterteeth.com/series/role-initiative', | |
306 | 'playlist_mincount': 16, | |
307 | 'info_dict': { | |
308 | 'id': 'role-initiative', | |
309 | 'title': 'Role Initiative', | |
8993721e | 310 | }, |
df03de2c M |
311 | }, { |
312 | 'url': 'https://roosterteeth.com/series/let-s-play-minecraft?season=9', | |
313 | 'playlist_mincount': 50, | |
314 | 'info_dict': { | |
315 | 'id': 'let-s-play-minecraft-9', | |
316 | 'title': 'Let\'s Play Minecraft - Season 9', | |
8993721e | 317 | }, |
244644c0 | 318 | }] |
319 | ||
320 | def _entries(self, series_id, season_number): | |
321 | display_id = join_nonempty(series_id, season_number) | |
8993721e B |
322 | |
323 | def yield_episodes(data): | |
324 | for episode in traverse_obj(data, ('data', lambda _, v: v['canonical_links']['self'])): | |
244644c0 | 325 | yield self.url_result( |
8993721e B |
326 | urljoin('https://www.roosterteeth.com', episode['canonical_links']['self']), |
327 | RoosterTeethIE, **self._extract_video_info(episode)) | |
328 | ||
329 | series_data = self._download_json( | |
330 | f'{self._API_BASE_URL}/shows/{series_id}/seasons?order=asc&order_by', display_id) | |
331 | for season_data in traverse_obj(series_data, ('data', lambda _, v: v['links']['episodes'])): | |
332 | idx = traverse_obj(season_data, ('attributes', 'number')) | |
333 | if season_number is not None and idx != season_number: | |
334 | continue | |
335 | yield from yield_episodes(self._download_json( | |
336 | urljoin(self._API_BASE, season_data['links']['episodes']), display_id, | |
337 | f'Downloading season {idx} JSON metadata', query={'per_page': 1000})) | |
338 | ||
339 | if season_number is None: # extract series-level bonus features | |
340 | yield from yield_episodes(self._download_json( | |
341 | f'{self._API_BASE_URL}/shows/{series_id}/bonus_features?order=asc&order_by&per_page=1000', | |
342 | display_id, 'Downloading bonus features JSON metadata', fatal=False)) | |
244644c0 | 343 | |
344 | def _real_extract(self, url): | |
345 | series_id = self._match_id(url) | |
346 | season_number = traverse_obj(parse_qs(url), ('season', 0), expected_type=int_or_none) | |
347 | ||
348 | entries = LazyList(self._entries(series_id, season_number)) | |
349 | return self.playlist_result( | |
350 | entries, | |
351 | join_nonempty(series_id, season_number), | |
352 | join_nonempty(entries[0].get('series'), season_number, delim=' - Season ')) |