2 from __future__
import unicode_literals
11 from ..compat
import (
33 class SoundcloudEmbedIE(InfoExtractor
):
34 _VALID_URL
= r
'https?://(?:w|player|p)\.soundcloud\.com/player/?.*?\burl=(?P<id>.+)'
36 # from https://www.soundi.fi/uutiset/ennakkokuuntelussa-timo-kaukolammen-station-to-station-to-station-julkaisua-juhlitaan-tanaan-g-livelabissa/
37 'url': 'https://w.soundcloud.com/player/?visual=true&url=https%3A%2F%2Fapi.soundcloud.com%2Fplaylists%2F922213810&show_artwork=true&maxwidth=640&maxheight=960&dnt=1&secret_token=s-ziYey',
38 'only_matching': True,
42 def _extract_urls(webpage
):
43 return [m
.group('url') for m
in re
.finditer(
44 r
'<iframe[^>]+src=(["\'])(?P
<url
>(?
:https?
://)?
(?
:w\
.)?soundcloud\
.com
/player
.+?
)\
1',
47 def _real_extract(self, url):
48 query = compat_urlparse.parse_qs(
49 compat_urlparse.urlparse(url).query)
50 api_url = query['url
'][0]
51 secret_token = query.get('secret_token
')
53 api_url = update_url_query(api_url, {'secret_token': secret_token[0]})
54 return self.url_result(api_url)
57 class SoundcloudIE(InfoExtractor):
58 """Information extractor for soundcloud.com
59 To access the media, the uid of the song and a stream token
60 must be extracted from the page source and the script must make
61 a request to media.soundcloud.com/crossdomain.xml. Then
62 the media can be grabbed by requesting from an url composed
63 of the stream token and uid
66 _VALID_URL = r'''(?x)^(?:https?://)?
67 (?:(?:(?:www\.|m\.)?soundcloud\.com/
69 (?P<uploader>[\w\d-]+)/
70 (?!(?:tracks|albums|sets(?:/.+?)?|reposts|likes|spotlight)/?(?:$|[?#]))
72 (?P<token>[^?]+?)?(?:[?].*)?$)
73 |(?:api(?:-v2)?\.soundcloud\.com/tracks/(?P<track_id>\d+)
74 (?:/?\?secret_token=(?P<secret_token>[^&]+))?)
77 IE_NAME = 'soundcloud
'
80 'url
': 'http
://soundcloud
.com
/ethmusic
/lostin
-powers
-she
-so
-heavy
',
81 'md5
': 'ebef0a451b909710ed1d7787dddbf0d7
',
85 'title
': 'Lostin Powers
- She so
Heavy (SneakPreview
) Adrian Ackers Blueprint
1',
86 'description
': 'No Downloads untill we record the finished version this weekend
, i was too pumped n i had to post it
, earl
is prolly gonna b hella p
.o
\'d
',
87 'uploader
': 'E
.T
. ExTerrestrial Music
',
88 'uploader_id
': '1571244',
89 'timestamp
': 1349920598,
90 'upload_date
': '20121011',
92 'license
': 'all
-rights
-reserved
',
101 'url
': 'https
://soundcloud
.com
/the
-concept
-band
/goldrushed
-mastered?
in=the
-concept
-band
/sets
/the
-royal
-concept
-ep
',
105 'title
': 'Goldrushed
',
106 'description
': 'From Stockholm Sweden
\r\nPovel
/ Magnus
/ Filip
/ David
\r\nwww
.theroyalconcept
.com
',
107 'uploader
': 'The Royal Concept
',
108 'uploader_id
': '9615865',
109 'timestamp
': 1337635207,
110 'upload_date
': '20120521',
112 'license
': 'all
-rights
-reserved
',
115 'comment_count
': int,
120 'skip_download
': True,
126 'url
': 'https
://soundcloud
.com
/jaimemf
/youtube
-dl
-test
-video
-a
-y
-baw
/s
-8Pjrp
',
127 'md5
': 'aa0dd32bfea9b0c5ef4f02aacd080604
',
131 'title
': 'Youtube
- Dl Test Video
\'\' Ä↭
',
132 'description
': 'test chars
: \"\'/\\ä↭
',
133 'uploader
': 'jaimeMF
',
134 'uploader_id
': '69767071',
135 'timestamp
': 1386604920,
136 'upload_date
': '20131209',
138 'license
': 'all
-rights
-reserved
',
141 'comment_count
': int,
145 # private link (alt format)
147 'url
': 'https
://api
.soundcloud
.com
/tracks
/123998367?secret_token
=s
-8Pjrp
',
148 'md5
': 'aa0dd32bfea9b0c5ef4f02aacd080604
',
152 'title
': 'Youtube
- Dl Test Video
\'\' Ä↭
',
153 'description
': 'test chars
: \"\'/\\ä↭
',
154 'uploader
': 'jaimeMF
',
155 'uploader_id
': '69767071',
156 'timestamp
': 1386604920,
157 'upload_date
': '20131209',
159 'license
': 'all
-rights
-reserved
',
162 'comment_count
': int,
168 'url
': 'https
://soundcloud
.com
/oddsamples
/bus
-brakes
',
169 'md5
': '7624f2351f8a3b2e7cd51522496e7631
',
173 'title
': 'Bus Brakes
',
174 'description
': 'md5
:0053ca6396e8d2fd7b7e1595ef12ab66
',
175 'uploader
': 'oddsamples
',
176 'uploader_id
': '73680509',
177 'timestamp
': 1389232924,
178 'upload_date
': '20140109',
180 'license
': 'cc
-by
-sa
',
183 'comment_count
': int,
187 # private link, downloadable format
189 'url
': 'https
://soundcloud
.com
/oriuplift
/uponly
-238-no
-talking
-wav
/s
-AyZUd
',
190 'md5
': '64a60b16e617d41d0bef032b7f55441e
',
194 'title
': 'Uplifting Only
238 [No Talking
] (incl
. Alex Feed Guestmix
) (Aug
31, 2017) [wav
]',
195 'description
': 'md5
:fa20ee0fca76a3d6df8c7e57f3715366
',
196 'uploader
': 'Ori Uplift Music
',
197 'uploader_id
': '12563093',
198 'timestamp
': 1504206263,
199 'upload_date
': '20170831',
200 'duration
': 7449.096,
201 'license
': 'all
-rights
-reserved
',
204 'comment_count
': int,
208 # no album art, use avatar pic for thumbnail
210 'url
': 'https
://soundcloud
.com
/garyvee
/sideways
-prod
-mad
-real
',
211 'md5
': '59c7872bc44e5d99b7211891664760c2
',
215 'title
': 'Sideways (Prod
. Mad Real
)',
216 'description
': 'md5
:d41d8cd98f00b204e9800998ecf8427e
',
217 'uploader
': 'garyvee
',
218 'uploader_id
': '2366352',
219 'timestamp
': 1488152409,
220 'upload_date
': '20170226',
222 'thumbnail
': r're
:https?
://.*\
.jpg
',
223 'license
': 'all
-rights
-reserved
',
226 'comment_count
': int,
230 'skip_download
': True,
233 # not available via api.soundcloud.com/i1/tracks/id/streams
235 'url
': 'https
://soundcloud
.com
/giovannisarani
/mezzo
-valzer
',
236 'md5
': 'e22aecd2bc88e0e4e432d7dcc0a1abf7
',
240 'title
': 'Mezzo Valzer
',
241 'description
': 'md5
:4138d582f81866a530317bae316e8b61
',
242 'uploader
': 'Giovanni Sarani
',
243 'uploader_id
': '3352531',
244 'timestamp
': 1551394171,
245 'upload_date
': '20190228',
247 'thumbnail
': r're
:https?
://.*\
.jpg
',
248 'license
': 'all
-rights
-reserved
',
251 'comment_count
': int,
254 'expected_warnings
': ['Unable to download JSON metadata
'],
258 _API_BASE = 'https
://api
.soundcloud
.com
/'
259 _API_V2_BASE = 'https
://api
-v2
.soundcloud
.com
/'
260 _BASE_URL = 'https
://soundcloud
.com
/'
261 _IMAGE_REPL_RE = r'-([0-9a
-z
]+)\
.jpg
'
276 def _store_client_id(self, client_id):
277 self._downloader.cache.store('soundcloud
', 'client_id
', client_id)
279 def _update_client_id(self):
280 webpage = self._download_webpage('https
://soundcloud
.com
/', None)
281 for src in reversed(re.findall(r'<script
[^
>]+src
="([^"]+)"', webpage)):
282 script = self._download_webpage(src, None, fatal=False)
284 client_id = self._search_regex(
285 r'client_id\s*:\s*"([0-9a
-zA
-Z
]{32}
)"',
286 script, 'client id', default=None)
288 self._CLIENT_ID = client_id
289 self._store_client_id(client_id)
291 raise ExtractorError('Unable to extract client id')
293 def _download_json(self, *args, **kwargs):
294 non_fatal = kwargs.get('fatal') is False
297 query = kwargs.get('query', {}).copy()
299 query['client_id'] = self._CLIENT_ID
300 kwargs['query'] = query
302 return super(SoundcloudIE, self)._download_json(*args, **compat_kwargs(kwargs))
303 except ExtractorError as e:
304 if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
305 self._store_client_id(None)
306 self._update_client_id()
309 self._downloader.report_warning(error_to_compat_str(e))
313 def _real_initialize(self):
314 self._CLIENT_ID = self._downloader.cache.load('soundcloud', 'client_id') or 'YUKXoArFcqrlQn9tfNHvvyfnDISj04zk'
317 def _resolv_url(cls, url):
318 return SoundcloudIE._API_V2_BASE + 'resolve?url=' + url
320 def _extract_info_dict(self, info, full_title=None, secret_token=None, version=2):
321 track_id = compat_str(info['id'])
322 title = info['title']
323 track_base_url = self._API_BASE + 'tracks/%s' % track_id
327 query = {'client_id': self._CLIENT_ID}
329 query['secret_token'] = secret_token
331 if info.get('downloadable') and info.get('has_downloads_left'):
332 format_url = update_url_query(
333 info.get('download_url') or track_base_url + '/download', query)
334 format_urls.add(format_url)
336 v1_info = self._download_json(
337 track_base_url, track_id, query=query, fatal=False) or {}
341 'format_id': 'download',
342 'ext': v1_info.get('original_format') or 'mp3',
343 'filesize': int_or_none(v1_info.get('original_content_size')),
348 def invalid_url(url):
349 return not url or url in format_urls or re.search(r'/(?:preview|playlist)/0/30/', url)
351 def add_format(f, protocol):
352 mobj = re.search(r'\.(?P<abr>\d+)\.(?P<ext>[0-9a-z]{3,4})(?=[/?])', stream_url)
354 for k, v in mobj.groupdict().items():
359 format_id_list.append(protocol)
360 for k in ('ext', 'abr'):
363 format_id_list.append(v)
368 'format_id': '_'.join(format_id_list),
369 'protocol': 'm3u8_native' if protocol == 'hls' else 'http',
374 transcodings = try_get(
375 info, lambda x: x['media']['transcodings'], list) or []
376 for t in transcodings:
377 if not isinstance(t, dict):
379 format_url = url_or_none(t.get('url'))
380 if not format_url or t.get('snipped') or '/preview/' in format_url:
382 stream = self._download_json(
383 format_url, track_id, query=query, fatal=False)
384 if not isinstance(stream, dict):
386 stream_url = url_or_none(stream.get('url'))
387 if invalid_url(stream_url):
389 format_urls.add(stream_url)
390 stream_format = t.get('format') or {}
391 protocol = stream_format.get('protocol')
392 if protocol != 'hls' and '/hls' in format_url:
395 preset = str_or_none(t.get('preset'))
397 ext = preset.split('_')[0]
398 if ext not in KNOWN_EXTENSIONS:
399 ext = mimetype2ext(stream_format.get('mime_type'))
403 }, 'http' if protocol == 'progressive' else protocol)
406 # Old API, does not work for some tracks (e.g.
407 # https://soundcloud.com/giovannisarani/mezzo-valzer)
408 # and might serve preview URLs (e.g.
409 # http://www.soundcloud.com/snbrn/ele)
410 format_dict = self._download_json(
411 track_base_url + '/streams', track_id,
412 'Downloading track url', query=query, fatal=False) or {}
414 for key, stream_url in format_dict.items():
415 if invalid_url(stream_url):
417 format_urls.add(stream_url)
418 mobj = re.search(r'(http|hls)_([^_]+)_(\d+)_url', key)
420 protocol, ext, abr = mobj.groups()
428 # We fallback to the stream_url in the original info, this
429 # cannot be always used, sometimes it can give an HTTP 404 error
430 urlh = self._request_webpage(
431 HEADRequest(info.get('stream_url') or track_base_url + '/stream'),
432 track_id, query=query, fatal=False)
434 stream_url = urlh.geturl()
435 if not invalid_url(stream_url):
436 add_format({'url': stream_url}, 'http')
441 self._sort_formats(formats)
443 user = info.get('user') or {}
446 artwork_url = info.get('artwork_url')
447 thumbnail = artwork_url or user.get('avatar_url')
448 if isinstance(thumbnail, compat_str):
449 if re.search(self._IMAGE_REPL_RE, thumbnail):
450 for image_id, size in self._ARTWORK_MAP.items():
453 'url': re.sub(self._IMAGE_REPL_RE, '-%s.jpg' % image_id, thumbnail),
455 if image_id == 'tiny' and not artwork_url:
457 elif image_id == 'original':
466 thumbnails = [{'url': thumbnail}]
468 def extract_count(key):
469 return int_or_none(info.get('%s_count' % key))
473 'uploader': user.get('username'),
474 'uploader_id': str_or_none(user.get('id')) or user.get('permalink'),
475 'uploader_url': user.get('permalink_url'),
476 'timestamp': unified_timestamp(info.get('created_at')),
478 'description': info.get('description'),
479 'thumbnails': thumbnails,
480 'duration': float_or_none(info.get('duration'), 1000),
481 'webpage_url': info.get('permalink_url'),
482 'license': info.get('license'),
483 'view_count': extract_count('playback'),
484 'like_count': extract_count('favoritings') or extract_count('likes'),
485 'comment_count': extract_count('comment'),
486 'repost_count': extract_count('reposts'),
487 'genre': info.get('genre'),
491 def _real_extract(self, url):
492 mobj = re.match(self._VALID_URL, url)
494 track_id = mobj.group('track_id')
498 info_json_url = self._API_V2_BASE + 'tracks/' + track_id
499 full_title = track_id
500 token = mobj.group('secret_token')
502 query['secret_token'] = token
504 full_title = resolve_title = '%s/%s' % mobj.group('uploader', 'title')
505 token = mobj.group('token')
507 resolve_title += '/%s' % token
508 info_json_url = self._resolv_url(self._BASE_URL + resolve_title)
511 info = self._download_json(
512 info_json_url, full_title, 'Downloading info JSON', query=query, fatal=False)
514 info = self._download_json(
515 info_json_url.replace(self._API_V2_BASE, self._API_BASE),
516 full_title, 'Downloading info JSON', query=query)
519 return self._extract_info_dict(info, full_title, token, version)
522 class SoundcloudPlaylistBaseIE(SoundcloudIE):
523 def _extract_track_entries(self, tracks, token=None):
526 track_id = str_or_none(track.get('id'))
527 url = track.get('permalink_url')
531 url = self._API_V2_BASE + 'tracks/' + track_id
533 url += '?secret_token=' + token
534 entries.append(self.url_result(
535 url, SoundcloudIE.ie_key(), track_id))
539 class SoundcloudSetIE(SoundcloudPlaylistBaseIE):
540 _VALID_URL = r'https?://(?:(?:www|m)\.)?soundcloud\.com/(?P<uploader>[\w\d-]+)/sets/(?P<slug_title>[\w\d-]+)(?:/(?P<token>[^?/]+))?'
541 IE_NAME = 'soundcloud:set'
543 'url': 'https://soundcloud.com/the-concept-band/sets/the-royal-concept-ep',
546 'title': 'The Royal Concept EP',
548 'playlist_mincount': 5,
550 'url': 'https://soundcloud.com/the-concept-band/sets/the-royal-concept-ep/token',
551 'only_matching': True,
554 def _real_extract(self, url):
555 mobj = re.match(self._VALID_URL, url)
557 full_title = '%s/sets/%s' % mobj.group('uploader', 'slug_title')
558 token = mobj.group('token')
560 full_title += '/' + token
562 info = self._download_json(self._resolv_url(
563 self._BASE_URL + full_title), full_title)
566 msgs = (compat_str(err['error_message']) for err in info['errors'])
567 raise ExtractorError('unable to download video webpage: %s' % ','.join(msgs))
569 entries = self._extract_track_entries(info['tracks'], token)
571 return self.playlist_result(
572 entries, str_or_none(info.get('id')), info.get('title'))
575 class SoundcloudPagedPlaylistBaseIE(SoundcloudPlaylistBaseIE):
576 def _extract_playlist(self, base_url, playlist_id, playlist_title):
579 'linked_partitioning': '1',
582 query = COMMON_QUERY.copy()
588 for i in itertools.count():
589 response = self._download_json(
590 next_href, playlist_id,
591 'Downloading track page %s' % (i + 1), query=query)
593 collection = response['collection']
595 if not isinstance(collection, list):
598 # Empty collection may be returned, in this case we proceed
599 # straight to next_href
601 def resolve_entry(candidates):
602 for cand in candidates:
603 if not isinstance(cand, dict):
605 permalink_url = url_or_none(cand.get('permalink_url'))
606 if not permalink_url:
608 return self.url_result(
610 SoundcloudIE.ie_key() if SoundcloudIE.suitable(permalink_url) else None,
611 str_or_none(cand.get('id')), cand.get('title'))
614 entry = resolve_entry((e, e.get('track'), e.get('playlist')))
616 entries.append(entry)
618 next_href = response.get('next_href')
622 next_href = response['next_href']
623 parsed_next_href = compat_urlparse.urlparse(next_href)
624 query = compat_urlparse.parse_qs(parsed_next_href.query)
625 query.update(COMMON_QUERY)
630 'title': playlist_title,
635 class SoundcloudUserIE(SoundcloudPagedPlaylistBaseIE):
636 _VALID_URL = r'''(?x)
638 (?:(?:www|m)\.)?soundcloud\.com/
641 (?P<rsrc>tracks|albums|sets|reposts|likes|spotlight)
645 IE_NAME = 'soundcloud:user'
647 'url': 'https://soundcloud.com/soft-cell-official',
650 'title': 'Soft Cell (All)',
652 'playlist_mincount': 28,
654 'url': 'https://soundcloud.com/soft-cell-official/tracks',
657 'title': 'Soft Cell (Tracks)',
659 'playlist_mincount': 27,
661 'url': 'https://soundcloud.com/soft-cell-official/albums',
664 'title': 'Soft Cell (Albums)',
666 'playlist_mincount': 1,
668 'url': 'https://soundcloud.com/jcv246/sets',
671 'title': 'Jordi / cv (Sets)',
673 'playlist_mincount': 2,
675 'url': 'https://soundcloud.com/jcv246/reposts',
678 'title': 'Jordi / cv (Reposts)',
680 'playlist_mincount': 6,
682 'url': 'https://soundcloud.com/clalberg/likes',
685 'title': 'clalberg (Likes)',
687 'playlist_mincount': 5,
689 'url': 'https://soundcloud.com/grynpyret/spotlight',
692 'title': 'Grynpyret (Spotlight)',
694 'playlist_mincount': 1,
698 'all': 'stream/users/%s',
699 'tracks': 'users/%s/tracks',
700 'albums': 'users/%s/albums',
701 'sets': 'users/%s/playlists',
702 'reposts': 'stream/users/%s/reposts',
703 'likes': 'users/%s/likes',
704 'spotlight': 'users/%s/spotlight',
707 def _real_extract(self, url):
708 mobj = re.match(self._VALID_URL, url)
709 uploader = mobj.group('user')
711 user = self._download_json(
712 self._resolv_url(self._BASE_URL + uploader),
713 uploader, 'Downloading user info')
715 resource = mobj.group('rsrc') or 'all'
717 return self._extract_playlist(
718 self._API_V2_BASE + self._BASE_URL_MAP[resource] % user['id'],
719 str_or_none(user.get('id')),
720 '%s (%s)' % (user['username'], resource.capitalize()))
723 class SoundcloudTrackStationIE(SoundcloudPagedPlaylistBaseIE):
724 _VALID_URL = r'https?://(?:(?:www|m)\.)?soundcloud\.com/stations/track/[^/]+/(?P<id>[^/?#&]+)'
725 IE_NAME = 'soundcloud:trackstation'
727 'url': 'https://soundcloud.com/stations/track/officialsundial/your-text',
730 'title': 'Track station: your text',
732 'playlist_mincount': 47,
735 def _real_extract(self, url):
736 track_name = self._match_id(url)
738 track = self._download_json(self._resolv_url(url), track_name)
739 track_id = self._search_regex(
740 r'soundcloud:track-stations:(\d+)', track['id'], 'track id')
742 return self._extract_playlist(
743 self._API_V2_BASE + 'stations/%s/tracks' % track['id'],
744 track_id, 'Track station: %s' % track['title'])
747 class SoundcloudPlaylistIE(SoundcloudPlaylistBaseIE):
748 _VALID_URL = r'https?://api(?:-v2)?\.soundcloud\.com/playlists/(?P<id>[0-9]+)(?:/?\?secret_token=(?P<token>[^&]+?))?$'
749 IE_NAME = 'soundcloud:playlist'
751 'url': 'https://api.soundcloud.com/playlists/4110309',
754 'title': 'TILT Brass - Bowery Poetry Club, August \'03 [Non-Site SCR 02]',
755 'description': 're:.*?TILT Brass - Bowery Poetry Club',
760 def _real_extract(self, url):
761 mobj = re.match(self._VALID_URL, url)
762 playlist_id = mobj.group('id')
765 token = mobj.group('token')
767 query['secret_token'] = token
769 data = self._download_json(
770 self._API_V2_BASE + 'playlists/' + playlist_id,
771 playlist_id, 'Downloading playlist', query=query)
773 entries = self._extract_track_entries(data['tracks'], token)
775 return self.playlist_result(
776 entries, playlist_id, data.get('title'), data.get('description'))
779 class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
780 IE_NAME = 'soundcloud:search'
781 IE_DESC = 'Soundcloud search'
782 _MAX_RESULTS = float('inf')
784 'url': 'scsearch15:post-avant jazzcore',
786 'title': 'post-avant jazzcore',
788 'playlist_count': 15,
791 _SEARCH_KEY = 'scsearch'
792 _MAX_RESULTS_PER_PAGE = 200
793 _DEFAULT_RESULTS_PER_PAGE = 50
795 def _get_collection(self, endpoint, collection_id, **query):
797 query.get('limit', self._DEFAULT_RESULTS_PER_PAGE),
798 self._MAX_RESULTS_PER_PAGE)
801 'linked_partitioning': 1,
804 next_url = update_url_query(self._API_V2_BASE + endpoint, query)
806 collected_results = 0
808 for i in itertools.count(1):
809 response = self._download_json(
810 next_url, collection_id, 'Downloading page {0}'.format(i),
811 'Unable to download API page')
813 collection = response.get('collection', [])
817 collection = list(filter(bool, collection))
818 collected_results += len(collection)
820 for item in collection:
821 yield self.url_result(item['uri'], SoundcloudIE.ie_key())
823 if not collection or collected_results >= limit:
826 next_url = response.get('next_href')
830 def _get_n_results(self, query, n):
831 tracks = self._get_collection('search/tracks', query, limit=n, q=query)
832 return self.playlist_result(tracks, playlist_title=query)