yt_dlp/extractor/roosterteeth.py

   1 from .common import InfoExtractor
   2 from ..compat import compat_HTTPError
   3 from ..utils import (
   4     ExtractorError,
   5     int_or_none,
   6     join_nonempty,
   7     LazyList,
   8     parse_qs,
   9     str_or_none,
  10     traverse_obj,
  11     url_or_none,
  12     urlencode_postdata,
  13     urljoin,
  14     update_url_query,
  15 )
  16
  17
  18 class RoosterTeethBaseIE(InfoExtractor):
  19     _NETRC_MACHINE = 'roosterteeth'
  20     _API_BASE = 'https://svod-be.roosterteeth.com'
  21     _API_BASE_URL = f'{_API_BASE}/api/v1'
  22
  23     def _perform_login(self, username, password):
  24         if self._get_cookies(self._API_BASE_URL).get('rt_access_token'):
  25             return
  26
  27         try:
  28             self._download_json(
  29                 'https://auth.roosterteeth.com/oauth/token',
  30                 None, 'Logging in', data=urlencode_postdata({
  31                     'client_id': '4338d2b4bdc8db1239360f28e72f0d9ddb1fd01e7a38fbb07b4b1f4ba4564cc5',
  32                     'grant_type': 'password',
  33                     'username': username,
  34                     'password': password,
  35                 }))
  36         except ExtractorError as e:
  37             msg = 'Unable to login'
  38             if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
  39                 resp = self._parse_json(e.cause.read().decode(), None, fatal=False)
  40                 if resp:
  41                     error = resp.get('extra_info') or resp.get('error_description') or resp.get('error')
  42                     if error:
  43                         msg += ': ' + error
  44             self.report_warning(msg)
  45
  46     def _extract_video_info(self, data):
  47         thumbnails = []
  48         for image in traverse_obj(data, ('included', 'images')):
  49             if image.get('type') not in ('episode_image', 'bonus_feature_image'):
  50                 continue
  51             thumbnails.extend([{
  52                 'id': name,
  53                 'url': url,
  54             } for name, url in (image.get('attributes') or {}).items() if url_or_none(url)])
  55
  56         attributes = data.get('attributes') or {}
  57         title = traverse_obj(attributes, 'title', 'display_title')
  58         sub_only = attributes.get('is_sponsors_only')
  59
  60         return {
  61             'id': str(data.get('id')),
  62             'display_id': attributes.get('slug'),
  63             'title': title,
  64             'description': traverse_obj(attributes, 'description', 'caption'),
  65             'series': attributes.get('show_title'),
  66             'season_number': int_or_none(attributes.get('season_number')),
  67             'season_id': attributes.get('season_id'),
  68             'episode': title,
  69             'episode_number': int_or_none(attributes.get('number')),
  70             'episode_id': str_or_none(data.get('uuid')),
  71             'channel_id': attributes.get('channel_id'),
  72             'duration': int_or_none(attributes.get('length')),
  73             'thumbnails': thumbnails,
  74             'availability': self._availability(
  75                 needs_premium=sub_only, needs_subscription=sub_only, needs_auth=sub_only,
  76                 is_private=False, is_unlisted=False),
  77             'tags': attributes.get('genres')
  78         }
  79
  80
  81 class RoosterTeethIE(RoosterTeethBaseIE):
  82     _VALID_URL = r'https?://(?:.+?\.)?roosterteeth\.com/(?:episode|watch)/(?P<id>[^/?#&]+)'
  83     _TESTS = [{
  84         'url': 'http://roosterteeth.com/episode/million-dollars-but-season-2-million-dollars-but-the-game-announcement',
  85         'info_dict': {
  86             'id': '9156',
  87             'display_id': 'million-dollars-but-season-2-million-dollars-but-the-game-announcement',
  88             'ext': 'mp4',
  89             'title': 'Million Dollars, But... The Game Announcement',
  90             'description': 'md5:168a54b40e228e79f4ddb141e89fe4f5',
  91             'thumbnail': r're:^https?://.*\.png$',
  92             'series': 'Million Dollars, But...',
  93             'episode': 'Million Dollars, But... The Game Announcement',
  94         },
  95         'params': {'skip_download': True},
  96     }, {
  97         'url': 'https://roosterteeth.com/watch/rwby-bonus-25',
  98         'info_dict': {
  99             'id': '40432',
 100             'display_id': 'rwby-bonus-25',
 101             'title': 'Grimm',
 102             'description': 'md5:f30ff570741213418a8d2c19868b93ab',
 103             'episode': 'Grimm',
 104             'channel_id': '92f780eb-ebfe-4bf5-a3b5-c6ad5460a5f1',
 105             'thumbnail': r're:^https?://.*\.(png|jpe?g)$',
 106             'ext': 'mp4',
 107         },
 108         'params': {'skip_download': True},
 109     }, {
 110         'url': 'http://achievementhunter.roosterteeth.com/episode/off-topic-the-achievement-hunter-podcast-2016-i-didn-t-think-it-would-pass-31',
 111         'only_matching': True,
 112     }, {
 113         'url': 'http://funhaus.roosterteeth.com/episode/funhaus-shorts-2016-austin-sucks-funhaus-shorts',
 114         'only_matching': True,
 115     }, {
 116         'url': 'http://screwattack.roosterteeth.com/episode/death-battle-season-3-mewtwo-vs-shadow',
 117         'only_matching': True,
 118     }, {
 119         'url': 'http://theknow.roosterteeth.com/episode/the-know-game-news-season-1-boring-steam-sales-are-better',
 120         'only_matching': True,
 121     }, {
 122         # only available for FIRST members
 123         'url': 'http://roosterteeth.com/episode/rt-docs-the-world-s-greatest-head-massage-the-world-s-greatest-head-massage-an-asmr-journey-part-one',
 124         'only_matching': True,
 125     }, {
 126         'url': 'https://roosterteeth.com/watch/million-dollars-but-season-2-million-dollars-but-the-game-announcement',
 127         'only_matching': True,
 128     }]
 129
 130     def _real_extract(self, url):
 131         display_id = self._match_id(url)
 132         api_episode_url = f'{self._API_BASE_URL}/watch/{display_id}'
 133
 134         try:
 135             video_data = self._download_json(
 136                 api_episode_url + '/videos', display_id,
 137                 'Downloading video JSON metadata')['data'][0]
 138             m3u8_url = video_data['attributes']['url']
 139             # XXX: additional URL at video_data['links']['download']
 140         except ExtractorError as e:
 141             if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
 142                 if self._parse_json(e.cause.read().decode(), display_id).get('access') is False:
 143                     self.raise_login_required(
 144                         '%s is only available for FIRST members' % display_id)
 145             raise
 146
 147         formats, subtitles = self._extract_m3u8_formats_and_subtitles(
 148             m3u8_url, display_id, 'mp4', 'm3u8_native', m3u8_id='hls')
 149
 150         episode = self._download_json(
 151             api_episode_url, display_id,
 152             'Downloading episode JSON metadata')['data'][0]
 153
 154         return {
 155             'display_id': display_id,
 156             'formats': formats,
 157             'subtitles': subtitles,
 158             **self._extract_video_info(episode)
 159         }
 160
 161
 162 class RoosterTeethSeriesIE(RoosterTeethBaseIE):
 163     _VALID_URL = r'https?://(?:.+?\.)?roosterteeth\.com/series/(?P<id>[^/?#&]+)'
 164     _TESTS = [{
 165         'url': 'https://roosterteeth.com/series/rwby?season=7',
 166         'playlist_count': 13,
 167         'info_dict': {
 168             'id': 'rwby-7',
 169             'title': 'RWBY - Season 7',
 170         }
 171     }, {
 172         'url': 'https://roosterteeth.com/series/role-initiative',
 173         'playlist_mincount': 16,
 174         'info_dict': {
 175             'id': 'role-initiative',
 176             'title': 'Role Initiative',
 177         }
 178     }, {
 179         'url': 'https://roosterteeth.com/series/let-s-play-minecraft?season=9',
 180         'playlist_mincount': 50,
 181         'info_dict': {
 182             'id': 'let-s-play-minecraft-9',
 183             'title': 'Let\'s Play Minecraft - Season 9',
 184         }
 185     }]
 186
 187     def _entries(self, series_id, season_number):
 188         display_id = join_nonempty(series_id, season_number)
 189         # TODO: extract bonus material
 190         for data in self._download_json(
 191                 f'{self._API_BASE_URL}/shows/{series_id}/seasons?order=asc&order_by', display_id)['data']:
 192             idx = traverse_obj(data, ('attributes', 'number'))
 193             if season_number and idx != season_number:
 194                 continue
 195             season_url = update_url_query(urljoin(self._API_BASE, data['links']['episodes']), {'per_page': 1000})
 196             season = self._download_json(season_url, display_id, f'Downloading season {idx} JSON metadata')['data']
 197             for episode in season:
 198                 yield self.url_result(
 199                     f'https://www.roosterteeth.com{episode["canonical_links"]["self"]}',
 200                     RoosterTeethIE.ie_key(),
 201                     **self._extract_video_info(episode))
 202
 203     def _real_extract(self, url):
 204         series_id = self._match_id(url)
 205         season_number = traverse_obj(parse_qs(url), ('season', 0), expected_type=int_or_none)
 206
 207         entries = LazyList(self._entries(series_id, season_number))
 208         return self.playlist_result(
 209             entries,
 210             join_nonempty(series_id, season_number),
 211             join_nonempty(entries[0].get('series'), season_number, delim=' - Season '))