yt_dlp/extractor/mixcloud.py

   1 from __future__ import unicode_literals
   2
   3 import itertools
   4
   5 from .common import InfoExtractor
   6 from ..compat import (
   7     compat_b64decode,
   8     compat_chr,
   9     compat_ord,
  10     compat_str,
  11     compat_urllib_parse_unquote,
  12     compat_zip
  13 )
  14 from ..utils import (
  15     ExtractorError,
  16     int_or_none,
  17     parse_iso8601,
  18     strip_or_none,
  19     try_get,
  20 )
  21
  22
  23 class MixcloudBaseIE(InfoExtractor):
  24     def _call_api(self, object_type, object_fields, display_id, username, slug=None):
  25         lookup_key = object_type + 'Lookup'
  26         return self._download_json(
  27             'https://www.mixcloud.com/graphql', display_id, query={
  28                 'query': '''{
  29   %s(lookup: {username: "%s"%s}) {
  30     %s
  31   }
  32 }''' % (lookup_key, username, ', slug: "%s"' % slug if slug else '', object_fields)
  33             })['data'][lookup_key]
  34
  35
  36 class MixcloudIE(MixcloudBaseIE):
  37     _VALID_URL = r'https?://(?:(?:www|beta|m)\.)?mixcloud\.com/([^/]+)/(?!stream|uploads|favorites|listens|playlists)([^/]+)'
  38     IE_NAME = 'mixcloud'
  39
  40     _TESTS = [{
  41         'url': 'http://www.mixcloud.com/dholbach/cryptkeeper/',
  42         'info_dict': {
  43             'id': 'dholbach_cryptkeeper',
  44             'ext': 'm4a',
  45             'title': 'Cryptkeeper',
  46             'description': 'After quite a long silence from myself, finally another Drum\'n\'Bass mix with my favourite current dance floor bangers.',
  47             'uploader': 'Daniel Holbach',
  48             'uploader_id': 'dholbach',
  49             'thumbnail': r're:https?://.*\.jpg',
  50             'view_count': int,
  51             'timestamp': 1321359578,
  52             'upload_date': '20111115',
  53         },
  54     }, {
  55         'url': 'http://www.mixcloud.com/gillespeterson/caribou-7-inch-vinyl-mix-chat/',
  56         'info_dict': {
  57             'id': 'gillespeterson_caribou-7-inch-vinyl-mix-chat',
  58             'ext': 'mp3',
  59             'title': 'Caribou 7 inch Vinyl Mix & Chat',
  60             'description': 'md5:2b8aec6adce69f9d41724647c65875e8',
  61             'uploader': 'Gilles Peterson Worldwide',
  62             'uploader_id': 'gillespeterson',
  63             'thumbnail': 're:https?://.*',
  64             'view_count': int,
  65             'timestamp': 1422987057,
  66             'upload_date': '20150203',
  67         },
  68     }, {
  69         'url': 'https://beta.mixcloud.com/RedLightRadio/nosedrip-15-red-light-radio-01-18-2016/',
  70         'only_matching': True,
  71     }]
  72     _DECRYPTION_KEY = 'IFYOUWANTTHEARTISTSTOGETPAIDDONOTDOWNLOADFROMMIXCLOUD'
  73
  74     @staticmethod
  75     def _decrypt_xor_cipher(key, ciphertext):
  76         """Encrypt/Decrypt XOR cipher. Both ways are possible because it's XOR."""
  77         return ''.join([
  78             compat_chr(compat_ord(ch) ^ compat_ord(k))
  79             for ch, k in compat_zip(ciphertext, itertools.cycle(key))])
  80
  81     def _real_extract(self, url):
  82         username, slug = self._match_valid_url(url).groups()
  83         username, slug = compat_urllib_parse_unquote(username), compat_urllib_parse_unquote(slug)
  84         track_id = '%s_%s' % (username, slug)
  85
  86         cloudcast = self._call_api('cloudcast', '''audioLength
  87     comments(first: 100) {
  88       edges {
  89         node {
  90           comment
  91           created
  92           user {
  93             displayName
  94             username
  95           }
  96         }
  97       }
  98       totalCount
  99     }
 100     description
 101     favorites {
 102       totalCount
 103     }
 104     featuringArtistList
 105     isExclusive
 106     name
 107     owner {
 108       displayName
 109       url
 110       username
 111     }
 112     picture(width: 1024, height: 1024) {
 113         url
 114     }
 115     plays
 116     publishDate
 117     reposts {
 118       totalCount
 119     }
 120     streamInfo {
 121       dashUrl
 122       hlsUrl
 123       url
 124     }
 125     tags {
 126       tag {
 127         name
 128       }
 129     }
 130     restrictedReason
 131     id''', track_id, username, slug)
 132
 133         if not cloudcast:
 134             raise ExtractorError('Track not found', expected=True)
 135
 136         reason = cloudcast.get('restrictedReason')
 137         if reason == 'tracklist':
 138             raise ExtractorError('Track unavailable in your country due to licensing restrictions', expected=True)
 139         elif reason == 'repeat_play':
 140             raise ExtractorError('You have reached your play limit for this track', expected=True)
 141         elif reason:
 142             raise ExtractorError('Track is restricted', expected=True)
 143
 144         title = cloudcast['name']
 145
 146         stream_info = cloudcast['streamInfo']
 147         formats = []
 148
 149         for url_key in ('url', 'hlsUrl', 'dashUrl'):
 150             format_url = stream_info.get(url_key)
 151             if not format_url:
 152                 continue
 153             decrypted = self._decrypt_xor_cipher(
 154                 self._DECRYPTION_KEY, compat_b64decode(format_url))
 155             if url_key == 'hlsUrl':
 156                 formats.extend(self._extract_m3u8_formats(
 157                     decrypted, track_id, 'mp4', entry_protocol='m3u8_native',
 158                     m3u8_id='hls', fatal=False))
 159             elif url_key == 'dashUrl':
 160                 formats.extend(self._extract_mpd_formats(
 161                     decrypted, track_id, mpd_id='dash', fatal=False))
 162             else:
 163                 formats.append({
 164                     'format_id': 'http',
 165                     'url': decrypted,
 166                     'downloader_options': {
 167                         # Mixcloud starts throttling at >~5M
 168                         'http_chunk_size': 5242880,
 169                     },
 170                 })
 171
 172         if not formats and cloudcast.get('isExclusive'):
 173             self.raise_login_required(metadata_available=True)
 174
 175         self._sort_formats(formats)
 176
 177         comments = []
 178         for edge in (try_get(cloudcast, lambda x: x['comments']['edges']) or []):
 179             node = edge.get('node') or {}
 180             text = strip_or_none(node.get('comment'))
 181             if not text:
 182                 continue
 183             user = node.get('user') or {}
 184             comments.append({
 185                 'author': user.get('displayName'),
 186                 'author_id': user.get('username'),
 187                 'text': text,
 188                 'timestamp': parse_iso8601(node.get('created')),
 189             })
 190
 191         tags = []
 192         for t in cloudcast.get('tags'):
 193             tag = try_get(t, lambda x: x['tag']['name'], compat_str)
 194             if not tag:
 195                 tags.append(tag)
 196
 197         get_count = lambda x: int_or_none(try_get(cloudcast, lambda y: y[x]['totalCount']))
 198
 199         owner = cloudcast.get('owner') or {}
 200
 201         return {
 202             'id': track_id,
 203             'title': title,
 204             'formats': formats,
 205             'description': cloudcast.get('description'),
 206             'thumbnail': try_get(cloudcast, lambda x: x['picture']['url'], compat_str),
 207             'uploader': owner.get('displayName'),
 208             'timestamp': parse_iso8601(cloudcast.get('publishDate')),
 209             'uploader_id': owner.get('username'),
 210             'uploader_url': owner.get('url'),
 211             'duration': int_or_none(cloudcast.get('audioLength')),
 212             'view_count': int_or_none(cloudcast.get('plays')),
 213             'like_count': get_count('favorites'),
 214             'repost_count': get_count('reposts'),
 215             'comment_count': get_count('comments'),
 216             'comments': comments,
 217             'tags': tags,
 218             'artist': ', '.join(cloudcast.get('featuringArtistList') or []) or None,
 219         }
 220
 221
 222 class MixcloudPlaylistBaseIE(MixcloudBaseIE):
 223     def _get_cloudcast(self, node):
 224         return node
 225
 226     def _get_playlist_title(self, title, slug):
 227         return title
 228
 229     def _real_extract(self, url):
 230         username, slug = self._match_valid_url(url).groups()
 231         username = compat_urllib_parse_unquote(username)
 232         if not slug:
 233             slug = 'uploads'
 234         else:
 235             slug = compat_urllib_parse_unquote(slug)
 236         playlist_id = '%s_%s' % (username, slug)
 237
 238         is_playlist_type = self._ROOT_TYPE == 'playlist'
 239         playlist_type = 'items' if is_playlist_type else slug
 240         list_filter = ''
 241
 242         has_next_page = True
 243         entries = []
 244         while has_next_page:
 245             playlist = self._call_api(
 246                 self._ROOT_TYPE, '''%s
 247     %s
 248     %s(first: 100%s) {
 249       edges {
 250         node {
 251           %s
 252         }
 253       }
 254       pageInfo {
 255         endCursor
 256         hasNextPage
 257       }
 258     }''' % (self._TITLE_KEY, self._DESCRIPTION_KEY, playlist_type, list_filter, self._NODE_TEMPLATE),
 259                 playlist_id, username, slug if is_playlist_type else None)
 260
 261             items = playlist.get(playlist_type) or {}
 262             for edge in items.get('edges', []):
 263                 cloudcast = self._get_cloudcast(edge.get('node') or {})
 264                 cloudcast_url = cloudcast.get('url')
 265                 if not cloudcast_url:
 266                     continue
 267                 slug = try_get(cloudcast, lambda x: x['slug'], compat_str)
 268                 owner_username = try_get(cloudcast, lambda x: x['owner']['username'], compat_str)
 269                 video_id = '%s_%s' % (owner_username, slug) if slug and owner_username else None
 270                 entries.append(self.url_result(
 271                     cloudcast_url, MixcloudIE.ie_key(), video_id))
 272
 273             page_info = items['pageInfo']
 274             has_next_page = page_info['hasNextPage']
 275             list_filter = ', after: "%s"' % page_info['endCursor']
 276
 277         return self.playlist_result(
 278             entries, playlist_id,
 279             self._get_playlist_title(playlist[self._TITLE_KEY], slug),
 280             playlist.get(self._DESCRIPTION_KEY))
 281
 282
 283 class MixcloudUserIE(MixcloudPlaylistBaseIE):
 284     _VALID_URL = r'https?://(?:www\.)?mixcloud\.com/(?P<id>[^/]+)/(?P<type>uploads|favorites|listens|stream)?/?$'
 285     IE_NAME = 'mixcloud:user'
 286
 287     _TESTS = [{
 288         'url': 'http://www.mixcloud.com/dholbach/',
 289         'info_dict': {
 290             'id': 'dholbach_uploads',
 291             'title': 'Daniel Holbach (uploads)',
 292             'description': 'md5:b60d776f0bab534c5dabe0a34e47a789',
 293         },
 294         'playlist_mincount': 36,
 295     }, {
 296         'url': 'http://www.mixcloud.com/dholbach/uploads/',
 297         'info_dict': {
 298             'id': 'dholbach_uploads',
 299             'title': 'Daniel Holbach (uploads)',
 300             'description': 'md5:b60d776f0bab534c5dabe0a34e47a789',
 301         },
 302         'playlist_mincount': 36,
 303     }, {
 304         'url': 'http://www.mixcloud.com/dholbach/favorites/',
 305         'info_dict': {
 306             'id': 'dholbach_favorites',
 307             'title': 'Daniel Holbach (favorites)',
 308             'description': 'md5:b60d776f0bab534c5dabe0a34e47a789',
 309         },
 310         # 'params': {
 311         #     'playlist_items': '1-100',
 312         # },
 313         'playlist_mincount': 396,
 314     }, {
 315         'url': 'http://www.mixcloud.com/dholbach/listens/',
 316         'info_dict': {
 317             'id': 'dholbach_listens',
 318             'title': 'Daniel Holbach (listens)',
 319             'description': 'md5:b60d776f0bab534c5dabe0a34e47a789',
 320         },
 321         # 'params': {
 322         #     'playlist_items': '1-100',
 323         # },
 324         'playlist_mincount': 1623,
 325         'skip': 'Large list',
 326     }, {
 327         'url': 'https://www.mixcloud.com/FirstEar/stream/',
 328         'info_dict': {
 329             'id': 'FirstEar_stream',
 330             'title': 'First Ear (stream)',
 331             'description': 'Curators of good music\r\n\r\nfirstearmusic.com',
 332         },
 333         'playlist_mincount': 271,
 334     }]
 335
 336     _TITLE_KEY = 'displayName'
 337     _DESCRIPTION_KEY = 'biog'
 338     _ROOT_TYPE = 'user'
 339     _NODE_TEMPLATE = '''slug
 340           url
 341           owner { username }'''
 342
 343     def _get_playlist_title(self, title, slug):
 344         return '%s (%s)' % (title, slug)
 345
 346
 347 class MixcloudPlaylistIE(MixcloudPlaylistBaseIE):
 348     _VALID_URL = r'https?://(?:www\.)?mixcloud\.com/(?P<user>[^/]+)/playlists/(?P<playlist>[^/]+)/?$'
 349     IE_NAME = 'mixcloud:playlist'
 350
 351     _TESTS = [{
 352         'url': 'https://www.mixcloud.com/maxvibes/playlists/jazzcat-on-ness-radio/',
 353         'info_dict': {
 354             'id': 'maxvibes_jazzcat-on-ness-radio',
 355             'title': 'Ness Radio sessions',
 356         },
 357         'playlist_mincount': 59,
 358     }]
 359     _TITLE_KEY = 'name'
 360     _DESCRIPTION_KEY = 'description'
 361     _ROOT_TYPE = 'playlist'
 362     _NODE_TEMPLATE = '''cloudcast {
 363             slug
 364             url
 365             owner { username }
 366           }'''
 367
 368     def _get_cloudcast(self, node):
 369         return node.get('cloudcast') or {}