yt_dlp/extractor/mixcloud.py

   1 from __future__ import unicode_literals
   2
   3 import itertools
   4
   5 from .common import InfoExtractor
   6 from ..compat import (
   7     compat_b64decode,
   8     compat_chr,
   9     compat_ord,
  10     compat_str,
  11     compat_urllib_parse_unquote,
  12     compat_zip
  13 )
  14 from ..utils import (
  15     int_or_none,
  16     parse_iso8601,
  17     strip_or_none,
  18     try_get,
  19 )
  20
  21
  22 class MixcloudBaseIE(InfoExtractor):
  23     def _call_api(self, object_type, object_fields, display_id, username, slug=None):
  24         lookup_key = object_type + 'Lookup'
  25         return self._download_json(
  26             'https://www.mixcloud.com/graphql', display_id, query={
  27                 'query': '''{
  28   %s(lookup: {username: "%s"%s}) {
  29     %s
  30   }
  31 }''' % (lookup_key, username, ', slug: "%s"' % slug if slug else '', object_fields)
  32             })['data'][lookup_key]
  33
  34
  35 class MixcloudIE(MixcloudBaseIE):
  36     _VALID_URL = r'https?://(?:(?:www|beta|m)\.)?mixcloud\.com/([^/]+)/(?!stream|uploads|favorites|listens|playlists)([^/]+)'
  37     IE_NAME = 'mixcloud'
  38
  39     _TESTS = [{
  40         'url': 'http://www.mixcloud.com/dholbach/cryptkeeper/',
  41         'info_dict': {
  42             'id': 'dholbach_cryptkeeper',
  43             'ext': 'm4a',
  44             'title': 'Cryptkeeper',
  45             'description': 'After quite a long silence from myself, finally another Drum\'n\'Bass mix with my favourite current dance floor bangers.',
  46             'uploader': 'Daniel Holbach',
  47             'uploader_id': 'dholbach',
  48             'thumbnail': r're:https?://.*\.jpg',
  49             'view_count': int,
  50             'timestamp': 1321359578,
  51             'upload_date': '20111115',
  52         },
  53     }, {
  54         'url': 'http://www.mixcloud.com/gillespeterson/caribou-7-inch-vinyl-mix-chat/',
  55         'info_dict': {
  56             'id': 'gillespeterson_caribou-7-inch-vinyl-mix-chat',
  57             'ext': 'mp3',
  58             'title': 'Caribou 7 inch Vinyl Mix & Chat',
  59             'description': 'md5:2b8aec6adce69f9d41724647c65875e8',
  60             'uploader': 'Gilles Peterson Worldwide',
  61             'uploader_id': 'gillespeterson',
  62             'thumbnail': 're:https?://.*',
  63             'view_count': int,
  64             'timestamp': 1422987057,
  65             'upload_date': '20150203',
  66         },
  67     }, {
  68         'url': 'https://beta.mixcloud.com/RedLightRadio/nosedrip-15-red-light-radio-01-18-2016/',
  69         'only_matching': True,
  70     }]
  71     _DECRYPTION_KEY = 'IFYOUWANTTHEARTISTSTOGETPAIDDONOTDOWNLOADFROMMIXCLOUD'
  72
  73     @staticmethod
  74     def _decrypt_xor_cipher(key, ciphertext):
  75         """Encrypt/Decrypt XOR cipher. Both ways are possible because it's XOR."""
  76         return ''.join([
  77             compat_chr(compat_ord(ch) ^ compat_ord(k))
  78             for ch, k in compat_zip(ciphertext, itertools.cycle(key))])
  79
  80     def _real_extract(self, url):
  81         username, slug = self._match_valid_url(url).groups()
  82         username, slug = compat_urllib_parse_unquote(username), compat_urllib_parse_unquote(slug)
  83         track_id = '%s_%s' % (username, slug)
  84
  85         cloudcast = self._call_api('cloudcast', '''audioLength
  86     comments(first: 100) {
  87       edges {
  88         node {
  89           comment
  90           created
  91           user {
  92             displayName
  93             username
  94           }
  95         }
  96       }
  97       totalCount
  98     }
  99     description
 100     favorites {
 101       totalCount
 102     }
 103     featuringArtistList
 104     isExclusive
 105     name
 106     owner {
 107       displayName
 108       url
 109       username
 110     }
 111     picture(width: 1024, height: 1024) {
 112         url
 113     }
 114     plays
 115     publishDate
 116     reposts {
 117       totalCount
 118     }
 119     streamInfo {
 120       dashUrl
 121       hlsUrl
 122       url
 123     }
 124     tags {
 125       tag {
 126         name
 127       }
 128     }''', track_id, username, slug)
 129
 130         title = cloudcast['name']
 131
 132         stream_info = cloudcast['streamInfo']
 133         formats = []
 134
 135         for url_key in ('url', 'hlsUrl', 'dashUrl'):
 136             format_url = stream_info.get(url_key)
 137             if not format_url:
 138                 continue
 139             decrypted = self._decrypt_xor_cipher(
 140                 self._DECRYPTION_KEY, compat_b64decode(format_url))
 141             if url_key == 'hlsUrl':
 142                 formats.extend(self._extract_m3u8_formats(
 143                     decrypted, track_id, 'mp4', entry_protocol='m3u8_native',
 144                     m3u8_id='hls', fatal=False))
 145             elif url_key == 'dashUrl':
 146                 formats.extend(self._extract_mpd_formats(
 147                     decrypted, track_id, mpd_id='dash', fatal=False))
 148             else:
 149                 formats.append({
 150                     'format_id': 'http',
 151                     'url': decrypted,
 152                     'downloader_options': {
 153                         # Mixcloud starts throttling at >~5M
 154                         'http_chunk_size': 5242880,
 155                     },
 156                 })
 157
 158         if not formats and cloudcast.get('isExclusive'):
 159             self.raise_login_required(metadata_available=True)
 160
 161         self._sort_formats(formats)
 162
 163         comments = []
 164         for edge in (try_get(cloudcast, lambda x: x['comments']['edges']) or []):
 165             node = edge.get('node') or {}
 166             text = strip_or_none(node.get('comment'))
 167             if not text:
 168                 continue
 169             user = node.get('user') or {}
 170             comments.append({
 171                 'author': user.get('displayName'),
 172                 'author_id': user.get('username'),
 173                 'text': text,
 174                 'timestamp': parse_iso8601(node.get('created')),
 175             })
 176
 177         tags = []
 178         for t in cloudcast.get('tags'):
 179             tag = try_get(t, lambda x: x['tag']['name'], compat_str)
 180             if not tag:
 181                 tags.append(tag)
 182
 183         get_count = lambda x: int_or_none(try_get(cloudcast, lambda y: y[x]['totalCount']))
 184
 185         owner = cloudcast.get('owner') or {}
 186
 187         return {
 188             'id': track_id,
 189             'title': title,
 190             'formats': formats,
 191             'description': cloudcast.get('description'),
 192             'thumbnail': try_get(cloudcast, lambda x: x['picture']['url'], compat_str),
 193             'uploader': owner.get('displayName'),
 194             'timestamp': parse_iso8601(cloudcast.get('publishDate')),
 195             'uploader_id': owner.get('username'),
 196             'uploader_url': owner.get('url'),
 197             'duration': int_or_none(cloudcast.get('audioLength')),
 198             'view_count': int_or_none(cloudcast.get('plays')),
 199             'like_count': get_count('favorites'),
 200             'repost_count': get_count('reposts'),
 201             'comment_count': get_count('comments'),
 202             'comments': comments,
 203             'tags': tags,
 204             'artist': ', '.join(cloudcast.get('featuringArtistList') or []) or None,
 205         }
 206
 207
 208 class MixcloudPlaylistBaseIE(MixcloudBaseIE):
 209     def _get_cloudcast(self, node):
 210         return node
 211
 212     def _get_playlist_title(self, title, slug):
 213         return title
 214
 215     def _real_extract(self, url):
 216         username, slug = self._match_valid_url(url).groups()
 217         username = compat_urllib_parse_unquote(username)
 218         if not slug:
 219             slug = 'uploads'
 220         else:
 221             slug = compat_urllib_parse_unquote(slug)
 222         playlist_id = '%s_%s' % (username, slug)
 223
 224         is_playlist_type = self._ROOT_TYPE == 'playlist'
 225         playlist_type = 'items' if is_playlist_type else slug
 226         list_filter = ''
 227
 228         has_next_page = True
 229         entries = []
 230         while has_next_page:
 231             playlist = self._call_api(
 232                 self._ROOT_TYPE, '''%s
 233     %s
 234     %s(first: 100%s) {
 235       edges {
 236         node {
 237           %s
 238         }
 239       }
 240       pageInfo {
 241         endCursor
 242         hasNextPage
 243       }
 244     }''' % (self._TITLE_KEY, self._DESCRIPTION_KEY, playlist_type, list_filter, self._NODE_TEMPLATE),
 245                 playlist_id, username, slug if is_playlist_type else None)
 246
 247             items = playlist.get(playlist_type) or {}
 248             for edge in items.get('edges', []):
 249                 cloudcast = self._get_cloudcast(edge.get('node') or {})
 250                 cloudcast_url = cloudcast.get('url')
 251                 if not cloudcast_url:
 252                     continue
 253                 slug = try_get(cloudcast, lambda x: x['slug'], compat_str)
 254                 owner_username = try_get(cloudcast, lambda x: x['owner']['username'], compat_str)
 255                 video_id = '%s_%s' % (owner_username, slug) if slug and owner_username else None
 256                 entries.append(self.url_result(
 257                     cloudcast_url, MixcloudIE.ie_key(), video_id))
 258
 259             page_info = items['pageInfo']
 260             has_next_page = page_info['hasNextPage']
 261             list_filter = ', after: "%s"' % page_info['endCursor']
 262
 263         return self.playlist_result(
 264             entries, playlist_id,
 265             self._get_playlist_title(playlist[self._TITLE_KEY], slug),
 266             playlist.get(self._DESCRIPTION_KEY))
 267
 268
 269 class MixcloudUserIE(MixcloudPlaylistBaseIE):
 270     _VALID_URL = r'https?://(?:www\.)?mixcloud\.com/(?P<id>[^/]+)/(?P<type>uploads|favorites|listens|stream)?/?$'
 271     IE_NAME = 'mixcloud:user'
 272
 273     _TESTS = [{
 274         'url': 'http://www.mixcloud.com/dholbach/',
 275         'info_dict': {
 276             'id': 'dholbach_uploads',
 277             'title': 'Daniel Holbach (uploads)',
 278             'description': 'md5:b60d776f0bab534c5dabe0a34e47a789',
 279         },
 280         'playlist_mincount': 36,
 281     }, {
 282         'url': 'http://www.mixcloud.com/dholbach/uploads/',
 283         'info_dict': {
 284             'id': 'dholbach_uploads',
 285             'title': 'Daniel Holbach (uploads)',
 286             'description': 'md5:b60d776f0bab534c5dabe0a34e47a789',
 287         },
 288         'playlist_mincount': 36,
 289     }, {
 290         'url': 'http://www.mixcloud.com/dholbach/favorites/',
 291         'info_dict': {
 292             'id': 'dholbach_favorites',
 293             'title': 'Daniel Holbach (favorites)',
 294             'description': 'md5:b60d776f0bab534c5dabe0a34e47a789',
 295         },
 296         # 'params': {
 297         #     'playlist_items': '1-100',
 298         # },
 299         'playlist_mincount': 396,
 300     }, {
 301         'url': 'http://www.mixcloud.com/dholbach/listens/',
 302         'info_dict': {
 303             'id': 'dholbach_listens',
 304             'title': 'Daniel Holbach (listens)',
 305             'description': 'md5:b60d776f0bab534c5dabe0a34e47a789',
 306         },
 307         # 'params': {
 308         #     'playlist_items': '1-100',
 309         # },
 310         'playlist_mincount': 1623,
 311         'skip': 'Large list',
 312     }, {
 313         'url': 'https://www.mixcloud.com/FirstEar/stream/',
 314         'info_dict': {
 315             'id': 'FirstEar_stream',
 316             'title': 'First Ear (stream)',
 317             'description': 'Curators of good music\r\n\r\nfirstearmusic.com',
 318         },
 319         'playlist_mincount': 271,
 320     }]
 321
 322     _TITLE_KEY = 'displayName'
 323     _DESCRIPTION_KEY = 'biog'
 324     _ROOT_TYPE = 'user'
 325     _NODE_TEMPLATE = '''slug
 326           url
 327           owner { username }'''
 328
 329     def _get_playlist_title(self, title, slug):
 330         return '%s (%s)' % (title, slug)
 331
 332
 333 class MixcloudPlaylistIE(MixcloudPlaylistBaseIE):
 334     _VALID_URL = r'https?://(?:www\.)?mixcloud\.com/(?P<user>[^/]+)/playlists/(?P<playlist>[^/]+)/?$'
 335     IE_NAME = 'mixcloud:playlist'
 336
 337     _TESTS = [{
 338         'url': 'https://www.mixcloud.com/maxvibes/playlists/jazzcat-on-ness-radio/',
 339         'info_dict': {
 340             'id': 'maxvibes_jazzcat-on-ness-radio',
 341             'title': 'Ness Radio sessions',
 342         },
 343         'playlist_mincount': 59,
 344     }]
 345     _TITLE_KEY = 'name'
 346     _DESCRIPTION_KEY = 'description'
 347     _ROOT_TYPE = 'playlist'
 348     _NODE_TEMPLATE = '''cloudcast {
 349             slug
 350             url
 351             owner { username }
 352           }'''
 353
 354     def _get_cloudcast(self, node):
 355         return node.get('cloudcast') or {}