yt_dlp/extractor/vevo.py

   1 import re
   2 import json
   3
   4 from .common import InfoExtractor
   5 from ..compat import (
   6     compat_str,
   7     compat_HTTPError,
   8 )
   9 from ..utils import (
  10     ExtractorError,
  11     int_or_none,
  12     parse_iso8601,
  13     parse_qs,
  14 )
  15
  16
  17 class VevoBaseIE(InfoExtractor):
  18     def _extract_json(self, webpage, video_id):
  19         return self._parse_json(
  20             self._search_regex(
  21                 r'window\.__INITIAL_STORE__\s*=\s*({.+?});\s*</script>',
  22                 webpage, 'initial store'),
  23             video_id)
  24
  25
  26 class VevoIE(VevoBaseIE):
  27     '''
  28     Accepts urls from vevo.com or in the format 'vevo:{id}'
  29     (currently used by MTVIE and MySpaceIE)
  30     '''
  31     _VALID_URL = r'''(?x)
  32         (?:https?://(?:www\.)?vevo\.com/watch/(?!playlist|genre)(?:[^/]+/(?:[^/]+/)?)?|
  33            https?://cache\.vevo\.com/m/html/embed\.html\?video=|
  34            https?://videoplayer\.vevo\.com/embed/embedded\?videoId=|
  35            https?://embed\.vevo\.com/.*?[?&]isrc=|
  36            vevo:)
  37         (?P<id>[^&?#]+)'''
  38
  39     _TESTS = []
  40     _VERSIONS = {
  41         0: 'youtube',  # only in AuthenticateVideo videoVersions
  42         1: 'level3',
  43         2: 'akamai',
  44         3: 'level3',
  45         4: 'amazon',
  46     }
  47
  48     def _initialize_api(self, video_id):
  49         webpage = self._download_webpage(
  50             'https://accounts.vevo.com/token', None,
  51             note='Retrieving oauth token',
  52             errnote='Unable to retrieve oauth token',
  53             data=json.dumps({
  54                 'client_id': 'SPupX1tvqFEopQ1YS6SS',
  55                 'grant_type': 'urn:vevo:params:oauth:grant-type:anonymous',
  56             }).encode('utf-8'),
  57             headers={
  58                 'Content-Type': 'application/json',
  59             })
  60
  61         if re.search(r'(?i)THIS PAGE IS CURRENTLY UNAVAILABLE IN YOUR REGION', webpage):
  62             self.raise_geo_restricted(
  63                 '%s said: This page is currently unavailable in your region' % self.IE_NAME)
  64
  65         auth_info = self._parse_json(webpage, video_id)
  66         self._api_url_template = self.http_scheme() + '//apiv2.vevo.com/%s?token=' + auth_info['legacy_token']
  67
  68     def _call_api(self, path, *args, **kwargs):
  69         try:
  70             data = self._download_json(self._api_url_template % path, *args, **kwargs)
  71         except ExtractorError as e:
  72             if isinstance(e.cause, compat_HTTPError):
  73                 errors = self._parse_json(e.cause.read().decode(), None)['errors']
  74                 error_message = ', '.join([error['message'] for error in errors])
  75                 raise ExtractorError('%s said: %s' % (self.IE_NAME, error_message), expected=True)
  76             raise
  77         return data
  78
  79     def _real_extract(self, url):
  80         video_id = self._match_id(url)
  81
  82         self._initialize_api(video_id)
  83
  84         video_info = self._call_api(
  85             'video/%s' % video_id, video_id, 'Downloading api video info',
  86             'Failed to download video info')
  87
  88         video_versions = self._call_api(
  89             'video/%s/streams' % video_id, video_id,
  90             'Downloading video versions info',
  91             'Failed to download video versions info',
  92             fatal=False)
  93
  94         # Some videos are only available via webpage (e.g.
  95         # https://github.com/ytdl-org/youtube-dl/issues/9366)
  96         if not video_versions:
  97             webpage = self._download_webpage(url, video_id)
  98             json_data = self._extract_json(webpage, video_id)
  99             if 'streams' in json_data.get('default', {}):
 100                 video_versions = json_data['default']['streams'][video_id][0]
 101             else:
 102                 video_versions = [
 103                     value
 104                     for key, value in json_data['apollo']['data'].items()
 105                     if key.startswith('%s.streams' % video_id)]
 106
 107         uploader = None
 108         artist = None
 109         featured_artist = None
 110         artists = video_info.get('artists')
 111         for curr_artist in artists:
 112             if curr_artist.get('role') == 'Featured':
 113                 featured_artist = curr_artist['name']
 114             else:
 115                 artist = uploader = curr_artist['name']
 116
 117         formats = []
 118         for video_version in video_versions:
 119             version = self._VERSIONS.get(video_version.get('version'), 'generic')
 120             version_url = video_version.get('url')
 121             if not version_url:
 122                 continue
 123
 124             if '.ism' in version_url:
 125                 continue
 126             elif '.mpd' in version_url:
 127                 formats.extend(self._extract_mpd_formats(
 128                     version_url, video_id, mpd_id='dash-%s' % version,
 129                     note='Downloading %s MPD information' % version,
 130                     errnote='Failed to download %s MPD information' % version,
 131                     fatal=False))
 132             elif '.m3u8' in version_url:
 133                 formats.extend(self._extract_m3u8_formats(
 134                     version_url, video_id, 'mp4', 'm3u8_native',
 135                     m3u8_id='hls-%s' % version,
 136                     note='Downloading %s m3u8 information' % version,
 137                     errnote='Failed to download %s m3u8 information' % version,
 138                     fatal=False))
 139             else:
 140                 m = re.search(r'''(?xi)
 141                     _(?P<width>[0-9]+)x(?P<height>[0-9]+)
 142                     _(?P<vcodec>[a-z0-9]+)
 143                     _(?P<vbr>[0-9]+)
 144                     _(?P<acodec>[a-z0-9]+)
 145                     _(?P<abr>[0-9]+)
 146                     \.(?P<ext>[a-z0-9]+)''', version_url)
 147                 if not m:
 148                     continue
 149
 150                 formats.append({
 151                     'url': version_url,
 152                     'format_id': 'http-%s-%s' % (version, video_version['quality']),
 153                     'vcodec': m.group('vcodec'),
 154                     'acodec': m.group('acodec'),
 155                     'vbr': int(m.group('vbr')),
 156                     'abr': int(m.group('abr')),
 157                     'ext': m.group('ext'),
 158                     'width': int(m.group('width')),
 159                     'height': int(m.group('height')),
 160                 })
 161         self._sort_formats(formats)
 162
 163         track = video_info['title']
 164         if featured_artist:
 165             artist = '%s ft. %s' % (artist, featured_artist)
 166         title = '%s - %s' % (artist, track) if artist else track
 167
 168         genres = video_info.get('genres')
 169         genre = (
 170             genres[0] if genres and isinstance(genres, list)
 171             and isinstance(genres[0], compat_str) else None)
 172
 173         is_explicit = video_info.get('isExplicit')
 174         if is_explicit is True:
 175             age_limit = 18
 176         elif is_explicit is False:
 177             age_limit = 0
 178         else:
 179             age_limit = None
 180
 181         return {
 182             'id': video_id,
 183             'title': title,
 184             'formats': formats,
 185             'thumbnail': video_info.get('imageUrl') or video_info.get('thumbnailUrl'),
 186             'timestamp': parse_iso8601(video_info.get('releaseDate')),
 187             'uploader': uploader,
 188             'duration': int_or_none(video_info.get('duration')),
 189             'view_count': int_or_none(video_info.get('views', {}).get('total')),
 190             'age_limit': age_limit,
 191             'track': track,
 192             'artist': uploader,
 193             'genre': genre,
 194         }
 195
 196
 197 class VevoPlaylistIE(VevoBaseIE):
 198     _VALID_URL = r'https?://(?:www\.)?vevo\.com/watch/(?P<kind>playlist|genre)/(?P<id>[^/?#&]+)'
 199
 200     _TESTS = [{
 201         'url': 'http://www.vevo.com/watch/genre/rock',
 202         'info_dict': {
 203             'id': 'rock',
 204             'title': 'Rock',
 205         },
 206         'playlist_count': 20,
 207     }, {
 208         'url': 'http://www.vevo.com/watch/genre/rock?index=0',
 209         'only_matching': True,
 210     }]
 211
 212     def _real_extract(self, url):
 213         mobj = self._match_valid_url(url)
 214         playlist_id = mobj.group('id')
 215         playlist_kind = mobj.group('kind')
 216
 217         webpage = self._download_webpage(url, playlist_id)
 218
 219         qs = parse_qs(url)
 220         index = qs.get('index', [None])[0]
 221
 222         if index:
 223             video_id = self._search_regex(
 224                 r'<meta[^>]+content=(["\'])vevo://video/(?P<id>.+?)\1[^>]*>',
 225                 webpage, 'video id', default=None, group='id')
 226             if video_id:
 227                 return self.url_result('vevo:%s' % video_id, VevoIE.ie_key())
 228
 229         playlists = self._extract_json(webpage, playlist_id)['default']['%ss' % playlist_kind]
 230
 231         playlist = (list(playlists.values())[0]
 232                     if playlist_kind == 'playlist' else playlists[playlist_id])
 233
 234         entries = [
 235             self.url_result('vevo:%s' % src, VevoIE.ie_key())
 236             for src in playlist['isrcs']]
 237
 238         return self.playlist_result(
 239             entries, playlist.get('playlistId') or playlist_id,
 240             playlist.get('name'), playlist.get('description'))