+# coding: utf-8
from __future__ import unicode_literals
import random
import time
from .common import InfoExtractor
-from ..compat import (
- compat_str,
- compat_urlparse,
-)
+from ..compat import compat_str
from ..utils import (
ExtractorError,
float_or_none,
parse_filesize,
str_or_none,
try_get,
- unescapeHTML,
update_url_query,
unified_strdate,
unified_timestamp,
url_or_none,
+ urljoin,
)
-class BandcampBaseIE(InfoExtractor):
- """Provide base functions for Bandcamp extractors"""
-
- def _extract_json_from_html_data_attribute(self, webpage, suffix, video_id):
- json_string = self._html_search_regex(
- r' data-%s="([^"]*)' % suffix,
- webpage, '%s json' % suffix, default='{}')
-
- return self._parse_json(json_string, video_id)
-
- def _parse_json_track(self, json):
- formats = []
- file_ = json.get('file')
- if isinstance(file_, dict):
- for format_id, format_url in file_.items():
- if not url_or_none(format_url):
- continue
- ext, abr_str = format_id.split('-', 1)
- formats.append({
- 'format_id': format_id,
- 'url': self._proto_relative_url(format_url, 'http:'),
- 'ext': ext,
- 'vcodec': 'none',
- 'acodec': ext,
- 'abr': int_or_none(abr_str),
- })
-
- return {
- 'duration': float_or_none(json.get('duration')),
- 'id': str_or_none(json.get('track_id') or json.get('id')),
- 'title': json.get('title'),
- 'title_link': json.get('title_link'),
- 'number': int_or_none(json.get('track_num')),
- 'formats': formats
- }
-
-
-class BandcampIE(BandcampBaseIE):
- IE_NAME = "Bandcamp:track"
- _VALID_URL = r'https?://[^/]+\.bandcamp\.com/track/(?P<title>[^/?#&]+)'
+class BandcampIE(InfoExtractor):
+ _VALID_URL = r'https?://[^/]+\.bandcamp\.com/track/(?P<id>[^/?#&]+)'
_TESTS = [{
'url': 'http://youtube-dl.bandcamp.com/track/youtube-dl-test-song',
'md5': 'c557841d5e50261777a6585648adf439',
'info_dict': {
'id': '1812978515',
'ext': 'mp3',
- 'title': "youtube-dl \"'/\\\u00e4\u21ad - youtube-dl \"'/\\\u00e4\u21ad - youtube-dl test song \"'/\\\u00e4\u21ad",
+ 'title': "youtube-dl \"'/\\ä↭ - youtube-dl \"'/\\ä↭ - youtube-dl test song \"'/\\ä↭",
'duration': 9.8485,
- 'uploader': "youtube-dl \"'/\\\u00e4\u21ad",
- 'timestamp': 1354224127,
+ 'uploader': 'youtube-dl "\'/\\ä↭',
'upload_date': '20121129',
+ 'timestamp': 1354224127,
},
'_skip': 'There is a limit of 200 free downloads / month for the test song'
}, {
# free download
'url': 'http://benprunty.bandcamp.com/track/lanius-battle',
- 'md5': '5d92af55811e47f38962a54c30b07ef0',
'info_dict': {
'id': '2650410135',
'ext': 'aiff',
},
}]
+ def _extract_data_attr(self, webpage, video_id, attr='tralbum', fatal=True):
+ return self._parse_json(self._html_search_regex(
+ r'data-%s=(["\'])({.+?})\1' % attr, webpage,
+ attr + ' data', group=2), video_id, fatal=fatal)
+
def _real_extract(self, url):
- mobj = re.match(self._VALID_URL, url)
- title = mobj.group('title')
- url_track_title = title
+ title = self._match_id(url)
webpage = self._download_webpage(url, title)
- thumbnail = self._html_search_meta('og:image', webpage, default=None)
-
- json_tralbum = self._extract_json_from_html_data_attribute(webpage, "tralbum", url_track_title)
- json_embed = self._extract_json_from_html_data_attribute(webpage, "embed", url_track_title)
-
- json_tracks = json_tralbum.get('trackinfo')
- if not json_tracks:
- raise ExtractorError('Could not extract track')
-
- track = self._parse_json_track(json_tracks[0])
- artist = json_tralbum.get('artist')
- album_title = json_embed.get('album_title')
-
- json_album = json_tralbum.get('packages')
- if json_album:
- json_album = json_album[0]
- album_publish_date = json_album.get('album_publish_date')
- album_release_date = json_album.get('album_release_date')
- else:
- album_publish_date = None
- album_release_date = json_tralbum.get('album_release_date')
-
- timestamp = unified_timestamp(json_tralbum.get('current', {}).get('publish_date') or album_publish_date)
- release_date = unified_strdate(album_release_date)
-
- download_link = self._search_regex(
- r'freeDownloadPage(?:["\']|"):\s*(["\']|")(?P<url>(?:(?!\1).)+)\1', webpage,
- 'download link', default=None, group='url')
+ tralbum = self._extract_data_attr(webpage, title)
+ thumbnail = self._og_search_thumbnail(webpage)
+
+ track_id = None
+ track = None
+ track_number = None
+ duration = None
+
+ formats = []
+ track_info = try_get(tralbum, lambda x: x['trackinfo'][0], dict)
+ if track_info:
+ file_ = track_info.get('file')
+ if isinstance(file_, dict):
+ for format_id, format_url in file_.items():
+ if not url_or_none(format_url):
+ continue
+ ext, abr_str = format_id.split('-', 1)
+ formats.append({
+ 'format_id': format_id,
+ 'url': self._proto_relative_url(format_url, 'http:'),
+ 'ext': ext,
+ 'vcodec': 'none',
+ 'acodec': ext,
+ 'abr': int_or_none(abr_str),
+ })
+ track = track_info.get('title')
+ track_id = str_or_none(
+ track_info.get('track_id') or track_info.get('id'))
+ track_number = int_or_none(track_info.get('track_num'))
+ duration = float_or_none(track_info.get('duration'))
+
+ embed = self._extract_data_attr(webpage, title, 'embed', False)
+ current = tralbum.get('current') or {}
+ artist = embed.get('artist') or current.get('artist') or tralbum.get('artist')
+ timestamp = unified_timestamp(
+ current.get('publish_date') or tralbum.get('album_publish_date'))
+
+ download_link = tralbum.get('freeDownloadPage')
if download_link:
- track_id = self._search_regex(
- r'\?id=(?P<id>\d+)&',
- download_link, 'track id')
+ track_id = compat_str(tralbum['id'])
download_webpage = self._download_webpage(
download_link, track_id, 'Downloading free downloads page')
- blob = self._parse_json(
- self._search_regex(
- r'data-blob=(["\'])(?P<blob>{.+?})\1', download_webpage,
- 'blob', group='blob'),
- track_id, transform_source=unescapeHTML)
+ blob = self._extract_data_attr(download_webpage, track_id, 'blob')
info = try_get(
blob, (lambda x: x['digital_items'][0],
if info:
downloads = info.get('downloads')
if isinstance(downloads, dict):
+ if not track:
+ track = info.get('title')
if not artist:
artist = info.get('artist')
if not thumbnail:
retry_url = url_or_none(stat.get('retry_url'))
if not retry_url:
continue
- track['formats'].append({
+ formats.append({
'url': self._proto_relative_url(retry_url, 'http:'),
'ext': download_formats.get(format_id),
'format_id': format_id,
'vcodec': 'none',
})
- self._sort_formats(track['formats'])
+ self._sort_formats(formats)
- title = '%s - %s' % (artist, track.get('title')) if artist else track.get('title')
+ title = '%s - %s' % (artist, track) if artist else track
+
+ if not duration:
+ duration = float_or_none(self._html_search_meta(
+ 'duration', webpage, default=None))
return {
- 'album': album_title,
- 'artist': artist,
- 'duration': track['duration'],
- 'formats': track['formats'],
- 'id': track['id'],
- 'release_date': release_date,
+ 'id': track_id,
+ 'title': title,
'thumbnail': thumbnail,
+ 'uploader': artist,
'timestamp': timestamp,
- 'title': title,
- 'track': track['title'],
- 'track_id': track['id'],
- 'track_number': track['number'],
- 'uploader': artist
+ 'release_date': unified_strdate(tralbum.get('album_release_date')),
+ 'duration': duration,
+ 'track': track,
+ 'track_number': track_number,
+ 'track_id': track_id,
+ 'artist': artist,
+ 'album': embed.get('album_title'),
+ 'formats': formats,
}
-class BandcampAlbumIE(BandcampBaseIE):
+class BandcampAlbumIE(BandcampIE):
IE_NAME = 'Bandcamp:album'
- _VALID_URL = r'https?://(?:(?P<subdomain>[^.]+)\.)?bandcamp\.com(?:/album/(?P<album_id>[^/?#&]+))?'
+ _VALID_URL = r'https?://(?:(?P<subdomain>[^.]+)\.)?bandcamp\.com(?:/album/(?P<id>[^/?#&]+))?'
_TESTS = [{
'url': 'http://blazo.bandcamp.com/album/jazz-format-mixtape-vol-1',
'info_dict': {
'id': '1353101989',
'ext': 'mp3',
- 'title': 'Intro',
+ 'title': 'Blazo - Intro',
+ 'timestamp': 1311756226,
+ 'upload_date': '20110727',
+ 'uploader': 'Blazo',
}
},
{
'info_dict': {
'id': '38097443',
'ext': 'mp3',
- 'title': 'Kero One - Keep It Alive (Blazo remix)',
+ 'title': 'Blazo - Kero One - Keep It Alive (Blazo remix)',
+ 'timestamp': 1311757238,
+ 'upload_date': '20110727',
+ 'uploader': 'Blazo',
}
},
],
'title': '"Entropy" EP',
'uploader_id': 'jstrecords',
'id': 'entropy-ep',
+ 'description': 'md5:0ff22959c943622972596062f2f366a5',
},
'playlist_mincount': 3,
}, {
'id': 'we-are-the-plague',
'title': 'WE ARE THE PLAGUE',
'uploader_id': 'insulters',
+ 'description': 'md5:b3cf845ee41b2b1141dc7bde9237255f',
},
'playlist_count': 2,
}]
else super(BandcampAlbumIE, cls).suitable(url))
def _real_extract(self, url):
- mobj = re.match(self._VALID_URL, url)
- uploader_id = mobj.group('subdomain')
- album_id = mobj.group('album_id')
+ uploader_id, album_id = re.match(self._VALID_URL, url).groups()
playlist_id = album_id or uploader_id
webpage = self._download_webpage(url, playlist_id)
-
- json_tralbum = self._extract_json_from_html_data_attribute(webpage, "tralbum", playlist_id)
- json_embed = self._extract_json_from_html_data_attribute(webpage, "embed", playlist_id)
-
- json_tracks = json_tralbum.get('trackinfo')
- if not json_tracks:
- raise ExtractorError('Could not extract album tracks')
-
- album_title = json_embed.get('album_title')
-
+ tralbum = self._extract_data_attr(webpage, playlist_id)
+ track_info = tralbum.get('trackinfo')
+ if not track_info:
+ raise ExtractorError('The page doesn\'t contain any tracks')
# Only tracks with duration info have songs
- tracks = [self._parse_json_track(track) for track in json_tracks]
entries = [
self.url_result(
- compat_urlparse.urljoin(url, track['title_link']),
- ie=BandcampIE.ie_key(), video_id=track['id'],
- video_title=track['title'])
- for track in tracks
- if track.get('duration')]
+ urljoin(url, t['title_link']), BandcampIE.ie_key(),
+ str_or_none(t.get('track_id') or t.get('id')), t.get('title'))
+ for t in track_info
+ if t.get('duration')]
+
+ current = tralbum.get('current') or {}
return {
'_type': 'playlist',
'uploader_id': uploader_id,
'id': playlist_id,
- 'title': album_title,
- 'entries': entries
+ 'title': current.get('title'),
+ 'description': current.get('about'),
+ 'entries': entries,
}
-class BandcampWeeklyIE(InfoExtractor):
+class BandcampWeeklyIE(BandcampIE):
IE_NAME = 'Bandcamp:weekly'
_VALID_URL = r'https?://(?:www\.)?bandcamp\.com/?\?(?:.*?&)?show=(?P<id>\d+)'
_TESTS = [{
'release_date': '20170404',
'series': 'Bandcamp Weekly',
'episode': 'Magic Moments',
- 'episode_number': 208,
'episode_id': '224',
- }
+ },
+ 'params': {
+ 'format': 'opus-lo',
+ },
}, {
'url': 'https://bandcamp.com/?blah/blah@&show=228',
'only_matching': True
}]
def _real_extract(self, url):
- video_id = self._match_id(url)
- webpage = self._download_webpage(url, video_id)
-
- blob = self._parse_json(
- self._search_regex(
- r'data-blob=(["\'])(?P<blob>{.+?})\1', webpage,
- 'blob', group='blob'),
- video_id, transform_source=unescapeHTML)
+ show_id = self._match_id(url)
+ webpage = self._download_webpage(url, show_id)
- show = blob['bcw_show']
+ blob = self._extract_data_attr(webpage, show_id, 'blob')
- # This is desired because any invalid show id redirects to `bandcamp.com`
- # which happens to expose the latest Bandcamp Weekly episode.
- show_id = int_or_none(show.get('show_id')) or int_or_none(video_id)
+ show = blob['bcw_data'][show_id]
formats = []
for format_id, format_url in show['audio_stream'].items():
if subtitle:
title += ' - %s' % subtitle
- episode_number = None
- seq = blob.get('bcw_seq')
-
- if seq and isinstance(seq, list):
- try:
- episode_number = next(
- int_or_none(e.get('episode_number'))
- for e in seq
- if isinstance(e, dict) and int_or_none(e.get('id')) == show_id)
- except StopIteration:
- pass
-
return {
- 'id': video_id,
+ 'id': show_id,
'title': title,
'description': show.get('desc') or show.get('short_desc'),
'duration': float_or_none(show.get('audio_duration')),
'release_date': unified_strdate(show.get('published_date')),
'series': 'Bandcamp Weekly',
'episode': show.get('subtitle'),
- 'episode_number': episode_number,
- 'episode_id': compat_str(video_id),
+ 'episode_id': show_id,
'formats': formats
}
projects / yt-dlp.git / blobdiff