yt_dlp/extractor/bandcamp.py

   1 import random
   2 import re
   3 import time
   4
   5 from .common import InfoExtractor
   6 from ..compat import compat_str
   7 from ..utils import (
   8     KNOWN_EXTENSIONS,
   9     ExtractorError,
  10     float_or_none,
  11     int_or_none,
  12     parse_filesize,
  13     str_or_none,
  14     try_get,
  15     unified_strdate,
  16     unified_timestamp,
  17     update_url_query,
  18     url_or_none,
  19     urljoin,
  20 )
  21
  22
  23 class BandcampIE(InfoExtractor):
  24     _VALID_URL = r'https?://(?P<uploader>[^/]+)\.bandcamp\.com/track/(?P<id>[^/?#&]+)'
  25     _EMBED_REGEX = [r'<meta property="og:url"[^>]*?content="(?P<url>.*?bandcamp\.com.*?)"']
  26     _TESTS = [{
  27         'url': 'http://youtube-dl.bandcamp.com/track/youtube-dl-test-song',
  28         'md5': 'c557841d5e50261777a6585648adf439',
  29         'info_dict': {
  30             'id': '1812978515',
  31             'ext': 'mp3',
  32             'title': 'youtube-dl "\'/\\ä↭ - youtube-dl "\'/\\ä↭ - youtube-dl test song "\'/\\ä↭',
  33             'duration': 9.8485,
  34             'uploader': 'youtube-dl "\'/\\ä↭',
  35             'upload_date': '20121129',
  36             'timestamp': 1354224127,
  37             'track': 'youtube-dl "\'/\\ä↭ - youtube-dl test song "\'/\\ä↭',
  38             'album_artist': 'youtube-dl "\'/\\ä↭',
  39             'track_id': '1812978515',
  40             'artist': 'youtube-dl "\'/\\ä↭',
  41             'uploader_url': 'https://youtube-dl.bandcamp.com',
  42             'uploader_id': 'youtube-dl',
  43             'thumbnail': 'https://f4.bcbits.com/img/a3216802731_5.jpg',
  44         },
  45         '_skip': 'There is a limit of 200 free downloads / month for the test song'
  46     }, {
  47         # free download
  48         'url': 'http://benprunty.bandcamp.com/track/lanius-battle',
  49         'info_dict': {
  50             'id': '2650410135',
  51             'ext': 'm4a',
  52             'acodec': r're:[fa]lac',
  53             'title': 'Ben Prunty - Lanius (Battle)',
  54             'thumbnail': r're:^https?://.*\.jpg$',
  55             'uploader': 'Ben Prunty',
  56             'timestamp': 1396508491,
  57             'upload_date': '20140403',
  58             'release_timestamp': 1396483200,
  59             'release_date': '20140403',
  60             'duration': 260.877,
  61             'track': 'Lanius (Battle)',
  62             'track_number': 1,
  63             'track_id': '2650410135',
  64             'artist': 'Ben Prunty',
  65             'album_artist': 'Ben Prunty',
  66             'album': 'FTL: Advanced Edition Soundtrack',
  67             'uploader_url': 'https://benprunty.bandcamp.com',
  68             'uploader_id': 'benprunty',
  69         },
  70     }, {
  71         # no free download, mp3 128
  72         'url': 'https://relapsealumni.bandcamp.com/track/hail-to-fire',
  73         'md5': 'fec12ff55e804bb7f7ebeb77a800c8b7',
  74         'info_dict': {
  75             'id': '2584466013',
  76             'ext': 'mp3',
  77             'title': 'Mastodon - Hail to Fire',
  78             'thumbnail': r're:^https?://.*\.jpg$',
  79             'uploader': 'Mastodon',
  80             'timestamp': 1322005399,
  81             'upload_date': '20111122',
  82             'release_timestamp': 1076112000,
  83             'release_date': '20040207',
  84             'duration': 120.79,
  85             'track': 'Hail to Fire',
  86             'track_number': 5,
  87             'track_id': '2584466013',
  88             'artist': 'Mastodon',
  89             'album_artist': 'Mastodon',
  90             'album': 'Call of the Mastodon',
  91             'uploader_url': 'https://relapsealumni.bandcamp.com',
  92             'uploader_id': 'relapsealumni',
  93         },
  94     }, {
  95         # track from compilation album (artist/album_artist difference)
  96         'url': 'https://diskotopia.bandcamp.com/track/safehouse',
  97         'md5': '19c5337bca1428afa54129f86a2f6a69',
  98         'info_dict': {
  99             'id': '1978174799',
 100             'ext': 'mp3',
 101             'title': 'submerse - submerse - Safehouse',
 102             'thumbnail': r're:^https?://.*\.jpg$',
 103             'uploader': 'submerse',
 104             'timestamp': 1480779297,
 105             'upload_date': '20161203',
 106             'release_timestamp': 1481068800,
 107             'release_date': '20161207',
 108             'duration': 154.066,
 109             'track': 'submerse - Safehouse',
 110             'track_number': 3,
 111             'track_id': '1978174799',
 112             'artist': 'submerse',
 113             'album_artist': 'Diskotopia',
 114             'album': 'DSK F/W 2016-2017 Free Compilation',
 115             'uploader_url': 'https://diskotopia.bandcamp.com',
 116             'uploader_id': 'diskotopia',
 117         },
 118     }]
 119
 120     def _extract_data_attr(self, webpage, video_id, attr='tralbum', fatal=True):
 121         return self._parse_json(self._html_search_regex(
 122             r'data-%s=(["\'])({.+?})\1' % attr, webpage,
 123             attr + ' data', group=2), video_id, fatal=fatal)
 124
 125     def _real_extract(self, url):
 126         title, uploader = self._match_valid_url(url).group('id', 'uploader')
 127         webpage = self._download_webpage(url, title)
 128         tralbum = self._extract_data_attr(webpage, title)
 129         thumbnail = self._og_search_thumbnail(webpage)
 130
 131         track_id = None
 132         track = None
 133         track_number = None
 134         duration = None
 135
 136         formats = []
 137         track_info = try_get(tralbum, lambda x: x['trackinfo'][0], dict)
 138         if track_info:
 139             file_ = track_info.get('file')
 140             if isinstance(file_, dict):
 141                 for format_id, format_url in file_.items():
 142                     if not url_or_none(format_url):
 143                         continue
 144                     ext, abr_str = format_id.split('-', 1)
 145                     formats.append({
 146                         'format_id': format_id,
 147                         'url': self._proto_relative_url(format_url, 'http:'),
 148                         'ext': ext,
 149                         'vcodec': 'none',
 150                         'acodec': ext,
 151                         'abr': int_or_none(abr_str),
 152                     })
 153             track = track_info.get('title')
 154             track_id = str_or_none(
 155                 track_info.get('track_id') or track_info.get('id'))
 156             track_number = int_or_none(track_info.get('track_num'))
 157             duration = float_or_none(track_info.get('duration'))
 158
 159         embed = self._extract_data_attr(webpage, title, 'embed', False)
 160         current = tralbum.get('current') or {}
 161         artist = embed.get('artist') or current.get('artist') or tralbum.get('artist')
 162         album_artist = self._html_search_regex(
 163             r'<h3 class="albumTitle">[\S\s]*?by\s*<span>\s*<a href="[^>]+">\s*([^>]+?)\s*</a>',
 164             webpage, 'album artist', fatal=False)
 165         timestamp = unified_timestamp(
 166             current.get('publish_date') or tralbum.get('album_publish_date'))
 167
 168         download_link = tralbum.get('freeDownloadPage')
 169         if download_link:
 170             track_id = compat_str(tralbum['id'])
 171
 172             download_webpage = self._download_webpage(
 173                 download_link, track_id, 'Downloading free downloads page')
 174
 175             blob = self._extract_data_attr(download_webpage, track_id, 'blob')
 176
 177             info = try_get(
 178                 blob, (lambda x: x['digital_items'][0],
 179                        lambda x: x['download_items'][0]), dict)
 180             if info:
 181                 downloads = info.get('downloads')
 182                 if isinstance(downloads, dict):
 183                     if not track:
 184                         track = info.get('title')
 185                     if not artist:
 186                         artist = info.get('artist')
 187                     if not thumbnail:
 188                         thumbnail = info.get('thumb_url')
 189
 190                     download_formats = {}
 191                     download_formats_list = blob.get('download_formats')
 192                     if isinstance(download_formats_list, list):
 193                         for f in blob['download_formats']:
 194                             name, ext = f.get('name'), f.get('file_extension')
 195                             if all(isinstance(x, compat_str) for x in (name, ext)):
 196                                 download_formats[name] = ext.strip('.')
 197
 198                     for format_id, f in downloads.items():
 199                         format_url = f.get('url')
 200                         if not format_url:
 201                             continue
 202                         # Stat URL generation algorithm is reverse engineered from
 203                         # download_*_bundle_*.js
 204                         stat_url = update_url_query(
 205                             format_url.replace('/download/', '/statdownload/'), {
 206                                 '.rand': int(time.time() * 1000 * random.random()),
 207                             })
 208                         format_id = f.get('encoding_name') or format_id
 209                         stat = self._download_json(
 210                             stat_url, track_id, 'Downloading %s JSON' % format_id,
 211                             transform_source=lambda s: s[s.index('{'):s.rindex('}') + 1],
 212                             fatal=False)
 213                         if not stat:
 214                             continue
 215                         retry_url = url_or_none(stat.get('retry_url'))
 216                         if not retry_url:
 217                             continue
 218                         formats.append({
 219                             'url': self._proto_relative_url(retry_url, 'http:'),
 220                             'ext': download_formats.get(format_id),
 221                             'format_id': format_id,
 222                             'format_note': f.get('description'),
 223                             'filesize': parse_filesize(f.get('size_mb')),
 224                             'vcodec': 'none',
 225                             'acodec': format_id.split('-')[0],
 226                         })
 227
 228         title = '%s - %s' % (artist, track) if artist else track
 229
 230         if not duration:
 231             duration = float_or_none(self._html_search_meta(
 232                 'duration', webpage, default=None))
 233
 234         return {
 235             'id': track_id,
 236             'title': title,
 237             'thumbnail': thumbnail,
 238             'uploader': artist,
 239             'uploader_id': uploader,
 240             'uploader_url': f'https://{uploader}.bandcamp.com',
 241             'timestamp': timestamp,
 242             'release_timestamp': unified_timestamp(tralbum.get('album_release_date')),
 243             'duration': duration,
 244             'track': track,
 245             'track_number': track_number,
 246             'track_id': track_id,
 247             'artist': artist,
 248             'album': embed.get('album_title'),
 249             'album_artist': album_artist,
 250             'formats': formats,
 251         }
 252
 253
 254 class BandcampAlbumIE(BandcampIE):  # XXX: Do not subclass from concrete IE
 255     IE_NAME = 'Bandcamp:album'
 256     _VALID_URL = r'https?://(?:(?P<subdomain>[^.]+)\.)?bandcamp\.com/album/(?P<id>[^/?#&]+)'
 257
 258     _TESTS = [{
 259         'url': 'http://blazo.bandcamp.com/album/jazz-format-mixtape-vol-1',
 260         'playlist': [
 261             {
 262                 'md5': '39bc1eded3476e927c724321ddf116cf',
 263                 'info_dict': {
 264                     'id': '1353101989',
 265                     'ext': 'mp3',
 266                     'title': 'Blazo - Intro',
 267                     'timestamp': 1311756226,
 268                     'upload_date': '20110727',
 269                     'uploader': 'Blazo',
 270                 }
 271             },
 272             {
 273                 'md5': '1a2c32e2691474643e912cc6cd4bffaa',
 274                 'info_dict': {
 275                     'id': '38097443',
 276                     'ext': 'mp3',
 277                     'title': 'Blazo - Kero One - Keep It Alive (Blazo remix)',
 278                     'timestamp': 1311757238,
 279                     'upload_date': '20110727',
 280                     'uploader': 'Blazo',
 281                 }
 282             },
 283         ],
 284         'info_dict': {
 285             'title': 'Jazz Format Mixtape vol.1',
 286             'id': 'jazz-format-mixtape-vol-1',
 287             'uploader_id': 'blazo',
 288         },
 289         'params': {
 290             'playlistend': 2
 291         },
 292         'skip': 'Bandcamp imposes download limits.'
 293     }, {
 294         'url': 'http://nightbringer.bandcamp.com/album/hierophany-of-the-open-grave',
 295         'info_dict': {
 296             'title': 'Hierophany of the Open Grave',
 297             'uploader_id': 'nightbringer',
 298             'id': 'hierophany-of-the-open-grave',
 299         },
 300         'playlist_mincount': 9,
 301     }, {
 302         # with escaped quote in title
 303         'url': 'https://jstrecords.bandcamp.com/album/entropy-ep',
 304         'info_dict': {
 305             'title': '"Entropy" EP',
 306             'uploader_id': 'jstrecords',
 307             'id': 'entropy-ep',
 308             'description': 'md5:0ff22959c943622972596062f2f366a5',
 309         },
 310         'playlist_mincount': 3,
 311     }, {
 312         # not all tracks have songs
 313         'url': 'https://insulters.bandcamp.com/album/we-are-the-plague',
 314         'info_dict': {
 315             'id': 'we-are-the-plague',
 316             'title': 'WE ARE THE PLAGUE',
 317             'uploader_id': 'insulters',
 318             'description': 'md5:b3cf845ee41b2b1141dc7bde9237255f',
 319         },
 320         'playlist_count': 2,
 321     }]
 322
 323     @classmethod
 324     def suitable(cls, url):
 325         return (False
 326                 if BandcampWeeklyIE.suitable(url) or BandcampIE.suitable(url)
 327                 else super(BandcampAlbumIE, cls).suitable(url))
 328
 329     def _real_extract(self, url):
 330         uploader_id, album_id = self._match_valid_url(url).groups()
 331         playlist_id = album_id or uploader_id
 332         webpage = self._download_webpage(url, playlist_id)
 333         tralbum = self._extract_data_attr(webpage, playlist_id)
 334         track_info = tralbum.get('trackinfo')
 335         if not track_info:
 336             raise ExtractorError('The page doesn\'t contain any tracks')
 337         # Only tracks with duration info have songs
 338         entries = [
 339             self.url_result(
 340                 urljoin(url, t['title_link']), BandcampIE.ie_key(),
 341                 str_or_none(t.get('track_id') or t.get('id')), t.get('title'))
 342             for t in track_info
 343             if t.get('duration')]
 344
 345         current = tralbum.get('current') or {}
 346
 347         return {
 348             '_type': 'playlist',
 349             'uploader_id': uploader_id,
 350             'id': playlist_id,
 351             'title': current.get('title'),
 352             'description': current.get('about'),
 353             'entries': entries,
 354         }
 355
 356
 357 class BandcampWeeklyIE(BandcampIE):  # XXX: Do not subclass from concrete IE
 358     IE_NAME = 'Bandcamp:weekly'
 359     _VALID_URL = r'https?://(?:www\.)?bandcamp\.com/?\?(?:.*?&)?show=(?P<id>\d+)'
 360     _TESTS = [{
 361         'url': 'https://bandcamp.com/?show=224',
 362         'md5': 'b00df799c733cf7e0c567ed187dea0fd',
 363         'info_dict': {
 364             'id': '224',
 365             'ext': 'opus',
 366             'title': 'BC Weekly April 4th 2017 - Magic Moments',
 367             'description': 'md5:5d48150916e8e02d030623a48512c874',
 368             'duration': 5829.77,
 369             'release_date': '20170404',
 370             'series': 'Bandcamp Weekly',
 371             'episode': 'Magic Moments',
 372             'episode_id': '224',
 373         },
 374         'params': {
 375             'format': 'opus-lo',
 376         },
 377     }, {
 378         'url': 'https://bandcamp.com/?blah/blah@&show=228',
 379         'only_matching': True
 380     }]
 381
 382     def _real_extract(self, url):
 383         show_id = self._match_id(url)
 384         webpage = self._download_webpage(url, show_id)
 385
 386         blob = self._extract_data_attr(webpage, show_id, 'blob')
 387
 388         show = blob['bcw_data'][show_id]
 389
 390         formats = []
 391         for format_id, format_url in show['audio_stream'].items():
 392             if not url_or_none(format_url):
 393                 continue
 394             for known_ext in KNOWN_EXTENSIONS:
 395                 if known_ext in format_id:
 396                     ext = known_ext
 397                     break
 398             else:
 399                 ext = None
 400             formats.append({
 401                 'format_id': format_id,
 402                 'url': format_url,
 403                 'ext': ext,
 404                 'vcodec': 'none',
 405             })
 406
 407         title = show.get('audio_title') or 'Bandcamp Weekly'
 408         subtitle = show.get('subtitle')
 409         if subtitle:
 410             title += ' - %s' % subtitle
 411
 412         return {
 413             'id': show_id,
 414             'title': title,
 415             'description': show.get('desc') or show.get('short_desc'),
 416             'duration': float_or_none(show.get('audio_duration')),
 417             'is_live': False,
 418             'release_date': unified_strdate(show.get('published_date')),
 419             'series': 'Bandcamp Weekly',
 420             'episode': show.get('subtitle'),
 421             'episode_id': show_id,
 422             'formats': formats
 423         }
 424
 425
 426 class BandcampUserIE(InfoExtractor):
 427     IE_NAME = 'Bandcamp:user'
 428     _VALID_URL = r'https?://(?!www\.)(?P<id>[^.]+)\.bandcamp\.com(?:/music)?/?(?:[#?]|$)'
 429
 430     _TESTS = [{
 431         # Type 1 Bandcamp user page.
 432         'url': 'https://adrianvonziegler.bandcamp.com',
 433         'info_dict': {
 434             'id': 'adrianvonziegler',
 435             'title': 'Discography of adrianvonziegler',
 436         },
 437         'playlist_mincount': 23,
 438     }, {
 439         # Bandcamp user page with only one album
 440         'url': 'http://dotscale.bandcamp.com',
 441         'info_dict': {
 442             'id': 'dotscale',
 443             'title': 'Discography of dotscale'
 444         },
 445         'playlist_count': 1,
 446     }, {
 447         # Type 2 Bandcamp user page.
 448         'url': 'https://nightcallofficial.bandcamp.com',
 449         'info_dict': {
 450             'id': 'nightcallofficial',
 451             'title': 'Discography of nightcallofficial',
 452         },
 453         'playlist_count': 4,
 454     }, {
 455         'url': 'https://steviasphere.bandcamp.com/music',
 456         'playlist_mincount': 47,
 457         'info_dict': {
 458             'id': 'steviasphere',
 459             'title': 'Discography of steviasphere',
 460         },
 461     }, {
 462         'url': 'https://coldworldofficial.bandcamp.com/music',
 463         'playlist_mincount': 10,
 464         'info_dict': {
 465             'id': 'coldworldofficial',
 466             'title': 'Discography of coldworldofficial',
 467         },
 468     }, {
 469         'url': 'https://nuclearwarnowproductions.bandcamp.com/music',
 470         'playlist_mincount': 399,
 471         'info_dict': {
 472             'id': 'nuclearwarnowproductions',
 473             'title': 'Discography of nuclearwarnowproductions',
 474         },
 475     }]
 476
 477     def _real_extract(self, url):
 478         uploader = self._match_id(url)
 479         webpage = self._download_webpage(url, uploader)
 480
 481         discography_data = (re.findall(r'<li data-item-id=["\'][^>]+>\s*<a href=["\'](?![^"\'/]*?/merch)([^"\']+)', webpage)
 482                             or re.findall(r'<div[^>]+trackTitle["\'][^"\']+["\']([^"\']+)', webpage))
 483
 484         return self.playlist_from_matches(
 485             discography_data, uploader, f'Discography of {uploader}', getter=lambda x: urljoin(url, x))