2 from __future__
import unicode_literals
11 from ..compat
import (
34 class SoundcloudEmbedIE(InfoExtractor
):
35 _VALID_URL
= r
'https?://(?:w|player|p)\.soundcloud\.com/player/?.*?\burl=(?P<id>.+)'
37 # from https://www.soundi.fi/uutiset/ennakkokuuntelussa-timo-kaukolammen-station-to-station-to-station-julkaisua-juhlitaan-tanaan-g-livelabissa/
38 'url': 'https://w.soundcloud.com/player/?visual=true&url=https%3A%2F%2Fapi.soundcloud.com%2Fplaylists%2F922213810&show_artwork=true&maxwidth=640&maxheight=960&dnt=1&secret_token=s-ziYey',
39 'only_matching': True,
43 def _extract_urls(webpage
):
44 return [m
.group('url') for m
in re
.finditer(
45 r
'<iframe[^>]+src=(["\'])(?P
<url
>(?
:https?
://)?
(?
:w\
.)?soundcloud\
.com
/player
.+?
)\
1',
48 def _real_extract(self, url):
49 query = compat_urlparse.parse_qs(
50 compat_urlparse.urlparse(url).query)
51 api_url = query['url
'][0]
52 secret_token = query.get('secret_token
')
54 api_url = update_url_query(api_url, {'secret_token': secret_token[0]})
55 return self.url_result(api_url)
58 class SoundcloudIE(InfoExtractor):
59 """Information extractor for soundcloud.com
60 To access the media, the uid of the song and a stream token
61 must be extracted from the page source and the script must make
62 a request to media.soundcloud.com/crossdomain.xml. Then
63 the media can be grabbed by requesting from an url composed
64 of the stream token and uid
67 _VALID_URL = r'''(?x)^(?:https?://)?
68 (?:(?:(?:www\.|m\.)?soundcloud\.com/
70 (?P<uploader>[\w\d-]+)/
71 (?!(?:tracks|albums|sets(?:/.+?)?|reposts|likes|spotlight)/?(?:$|[?#]))
73 (?P<token>[^?]+?)?(?:[?].*)?$)
74 |(?:api(?:-v2)?\.soundcloud\.com/tracks/(?P<track_id>\d+)
75 (?:/?\?secret_token=(?P<secret_token>[^&]+))?)
78 IE_NAME = 'soundcloud
'
81 'url
': 'http
://soundcloud
.com
/ethmusic
/lostin
-powers
-she
-so
-heavy
',
82 'md5
': 'ebef0a451b909710ed1d7787dddbf0d7
',
86 'title
': 'Lostin Powers
- She so
Heavy (SneakPreview
) Adrian Ackers Blueprint
1',
87 'description
': 'No Downloads untill we record the finished version this weekend
, i was too pumped n i had to post it
, earl
is prolly gonna b hella p
.o
\'d
',
88 'uploader
': 'E
.T
. ExTerrestrial Music
',
89 'uploader_id
': '1571244',
90 'timestamp
': 1349920598,
91 'upload_date
': '20121011',
93 'license
': 'all
-rights
-reserved
',
102 'url
': 'https
://soundcloud
.com
/the
-concept
-band
/goldrushed
-mastered?
in=the
-concept
-band
/sets
/the
-royal
-concept
-ep
',
106 'title
': 'Goldrushed
',
107 'description
': 'From Stockholm Sweden
\r\nPovel
/ Magnus
/ Filip
/ David
\r\nwww
.theroyalconcept
.com
',
108 'uploader
': 'The Royal Concept
',
109 'uploader_id
': '9615865',
110 'timestamp
': 1337635207,
111 'upload_date
': '20120521',
113 'license
': 'all
-rights
-reserved
',
116 'comment_count
': int,
122 'url
': 'https
://soundcloud
.com
/jaimemf
/youtube
-dl
-test
-video
-a
-y
-baw
/s
-8Pjrp
',
123 'md5
': 'aa0dd32bfea9b0c5ef4f02aacd080604
',
127 'title
': 'Youtube
- Dl Test Video
\'\' Ä↭
',
128 'description
': 'test chars
: \"\'/\\ä↭
',
129 'uploader
': 'jaimeMF
',
130 'uploader_id
': '69767071',
131 'timestamp
': 1386604920,
132 'upload_date
': '20131209',
134 'license
': 'all
-rights
-reserved
',
137 'comment_count
': int,
141 # private link (alt format)
143 'url
': 'https
://api
.soundcloud
.com
/tracks
/123998367?secret_token
=s
-8Pjrp
',
144 'md5
': 'aa0dd32bfea9b0c5ef4f02aacd080604
',
148 'title
': 'Youtube
- Dl Test Video
\'\' Ä↭
',
149 'description
': 'test chars
: \"\'/\\ä↭
',
150 'uploader
': 'jaimeMF
',
151 'uploader_id
': '69767071',
152 'timestamp
': 1386604920,
153 'upload_date
': '20131209',
155 'license
': 'all
-rights
-reserved
',
158 'comment_count
': int,
164 'url
': 'https
://soundcloud
.com
/oddsamples
/bus
-brakes
',
165 'md5
': '7624f2351f8a3b2e7cd51522496e7631
',
169 'title
': 'Bus Brakes
',
170 'description
': 'md5
:0053ca6396e8d2fd7b7e1595ef12ab66
',
171 'uploader
': 'oddsamples
',
172 'uploader_id
': '73680509',
173 'timestamp
': 1389232924,
174 'upload_date
': '20140109',
176 'license
': 'cc
-by
-sa
',
179 'comment_count
': int,
183 # private link, downloadable format
185 'url
': 'https
://soundcloud
.com
/oriuplift
/uponly
-238-no
-talking
-wav
/s
-AyZUd
',
186 'md5
': '64a60b16e617d41d0bef032b7f55441e
',
190 'title
': 'Uplifting Only
238 [No Talking
] (incl
. Alex Feed Guestmix
) (Aug
31, 2017) [wav
]',
191 'description
': 'md5
:fa20ee0fca76a3d6df8c7e57f3715366
',
192 'uploader
': 'Ori Uplift Music
',
193 'uploader_id
': '12563093',
194 'timestamp
': 1504206263,
195 'upload_date
': '20170831',
196 'duration
': 7449.096,
197 'license
': 'all
-rights
-reserved
',
200 'comment_count
': int,
204 # no album art, use avatar pic for thumbnail
206 'url
': 'https
://soundcloud
.com
/garyvee
/sideways
-prod
-mad
-real
',
207 'md5
': '59c7872bc44e5d99b7211891664760c2
',
211 'title
': 'Sideways (Prod
. Mad Real
)',
212 'description
': 'md5
:d41d8cd98f00b204e9800998ecf8427e
',
213 'uploader
': 'garyvee
',
214 'uploader_id
': '2366352',
215 'timestamp
': 1488152409,
216 'upload_date
': '20170226',
218 'thumbnail
': r're
:https?
://.*\
.jpg
',
219 'license
': 'all
-rights
-reserved
',
222 'comment_count
': int,
226 'skip_download
': True,
230 'url
': 'https
://soundcloud
.com
/giovannisarani
/mezzo
-valzer
',
231 'md5
': 'e22aecd2bc88e0e4e432d7dcc0a1abf7
',
235 'title
': 'Mezzo Valzer
',
236 'description
': 'md5
:4138d582f81866a530317bae316e8b61
',
237 'uploader
': 'Micronie
',
238 'uploader_id
': '3352531',
239 'timestamp
': 1551394171,
240 'upload_date
': '20190228',
242 'thumbnail
': r're
:https?
://.*\
.jpg
',
243 'license
': 'all
-rights
-reserved
',
246 'comment_count
': int,
251 # with AAC HQ format available via OAuth token
252 'url
': 'https
://soundcloud
.com
/wandw
/the
-chainsmokers
-ft
-daya
-dont
-let
-me
-down
-ww
-remix
-1',
253 'only_matching
': True,
257 _API_V2_BASE = 'https
://api
-v2
.soundcloud
.com
/'
258 _BASE_URL = 'https
://soundcloud
.com
/'
259 _IMAGE_REPL_RE = r'-([0-9a
-z
]+)\
.jpg
'
274 def _store_client_id(self, client_id):
275 self._downloader.cache.store('soundcloud
', 'client_id
', client_id)
277 def _update_client_id(self):
278 webpage = self._download_webpage('https
://soundcloud
.com
/', None)
279 for src in reversed(re.findall(r'<script
[^
>]+src
="([^"]+)"', webpage)):
280 script = self._download_webpage(src, None, fatal=False)
282 client_id = self._search_regex(
283 r'client_id\s*:\s*"([0-9a
-zA
-Z
]{32}
)"',
284 script, 'client id', default=None)
286 self._CLIENT_ID = client_id
287 self._store_client_id(client_id)
289 raise ExtractorError('Unable to extract client id')
291 def _download_json(self, *args, **kwargs):
292 non_fatal = kwargs.get('fatal') is False
295 query = kwargs.get('query', {}).copy()
297 query['client_id'] = self._CLIENT_ID
298 kwargs['query'] = query
300 return super(SoundcloudIE, self)._download_json(*args, **compat_kwargs(kwargs))
301 except ExtractorError as e:
302 if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
303 self._store_client_id(None)
304 self._update_client_id()
307 self._downloader.report_warning(error_to_compat_str(e))
311 def _real_initialize(self):
312 self._CLIENT_ID = self._downloader.cache.load('soundcloud', 'client_id') or 'YUKXoArFcqrlQn9tfNHvvyfnDISj04zk'
315 def _resolv_url(cls, url):
316 return SoundcloudIE._API_V2_BASE + 'resolve?url=' + url
318 def _extract_info_dict(self, info, full_title=None, secret_token=None):
319 track_id = compat_str(info['id'])
320 title = info['title']
324 query = {'client_id': self._CLIENT_ID}
326 query['secret_token'] = secret_token
328 if info.get('downloadable') and info.get('has_downloads_left'):
329 download_url = update_url_query(
330 self._API_V2_BASE + 'tracks/' + track_id + '/download', query)
331 redirect_url = (self._download_json(download_url, track_id, fatal=False) or {}).get('redirectUri')
333 urlh = self._request_webpage(
334 HEADRequest(redirect_url), track_id, fatal=False)
336 format_url = urlh.geturl()
337 format_urls.add(format_url)
339 'format_id': 'download',
340 'ext': urlhandle_detect_ext(urlh) or 'mp3',
341 'filesize': int_or_none(urlh.headers.get('Content-Length')),
346 def invalid_url(url):
347 return not url or url in format_urls
349 def add_format(f, protocol, is_preview=False):
350 mobj = re.search(r'\.(?P<abr>\d+)\.(?P<ext>[0-9a-z]{3,4})(?=[/?])', stream_url)
352 for k, v in mobj.groupdict().items():
357 format_id_list.append(protocol)
361 for k in ('ext', 'abr'):
364 format_id_list.append(v)
365 preview = is_preview or re.search(r'/(?:preview|playlist)/0/30/', f['url'])
367 format_id_list.append('preview')
371 if protocol == 'hls':
372 protocol = 'm3u8' if ext == 'aac' else 'm3u8_native'
376 'format_id': '_'.join(format_id_list),
377 'protocol': protocol,
378 'preference': -10 if preview else None,
383 transcodings = try_get(
384 info, lambda x: x['media']['transcodings'], list) or []
385 for t in transcodings:
386 if not isinstance(t, dict):
388 format_url = url_or_none(t.get('url'))
391 stream = self._download_json(
392 format_url, track_id, query=query, fatal=False)
393 if not isinstance(stream, dict):
395 stream_url = url_or_none(stream.get('url'))
396 if invalid_url(stream_url):
398 format_urls.add(stream_url)
399 stream_format = t.get('format') or {}
400 protocol = stream_format.get('protocol')
401 if protocol != 'hls' and '/hls' in format_url:
404 preset = str_or_none(t.get('preset'))
406 ext = preset.split('_')[0]
407 if ext not in KNOWN_EXTENSIONS:
408 ext = mimetype2ext(stream_format.get('mime_type'))
412 }, 'http' if protocol == 'progressive' else protocol,
413 t.get('snipped') or '/preview/' in format_url)
418 if not formats and info.get('policy') == 'BLOCK':
419 self.raise_geo_restricted()
420 self._sort_formats(formats)
422 user = info.get('user') or {}
425 artwork_url = info.get('artwork_url')
426 thumbnail = artwork_url or user.get('avatar_url')
427 if isinstance(thumbnail, compat_str):
428 if re.search(self._IMAGE_REPL_RE, thumbnail):
429 for image_id, size in self._ARTWORK_MAP.items():
432 'url': re.sub(self._IMAGE_REPL_RE, '-%s.jpg' % image_id, thumbnail),
434 if image_id == 'tiny' and not artwork_url:
436 elif image_id == 'original':
445 thumbnails = [{'url': thumbnail}]
447 def extract_count(key):
448 return int_or_none(info.get('%s_count' % key))
452 'uploader': user.get('username'),
453 'uploader_id': str_or_none(user.get('id')) or user.get('permalink'),
454 'uploader_url': user.get('permalink_url'),
455 'timestamp': unified_timestamp(info.get('created_at')),
457 'description': info.get('description'),
458 'thumbnails': thumbnails,
459 'duration': float_or_none(info.get('duration'), 1000),
460 'webpage_url': info.get('permalink_url'),
461 'license': info.get('license'),
462 'view_count': extract_count('playback'),
463 'like_count': extract_count('favoritings') or extract_count('likes'),
464 'comment_count': extract_count('comment'),
465 'repost_count': extract_count('reposts'),
466 'genre': info.get('genre'),
470 def _real_extract(self, url):
471 mobj = re.match(self._VALID_URL, url)
473 track_id = mobj.group('track_id')
477 info_json_url = self._API_V2_BASE + 'tracks/' + track_id
478 full_title = track_id
479 token = mobj.group('secret_token')
481 query['secret_token'] = token
483 full_title = resolve_title = '%s/%s' % mobj.group('uploader', 'title')
484 token = mobj.group('token')
486 resolve_title += '/%s' % token
487 info_json_url = self._resolv_url(self._BASE_URL + resolve_title)
489 info = self._download_json(
490 info_json_url, full_title, 'Downloading info JSON', query=query)
492 return self._extract_info_dict(info, full_title, token)
495 class SoundcloudPlaylistBaseIE(SoundcloudIE):
496 def _extract_set(self, playlist, token=None):
497 playlist_id = compat_str(playlist['id'])
498 tracks = playlist.get('tracks') or []
499 if not all([t.get('permalink_url') for t in tracks]) and token:
500 tracks = self._download_json(
501 self._API_V2_BASE + 'tracks', playlist_id,
502 'Downloading tracks', query={
503 'ids': ','.join([compat_str(t['id']) for t in tracks]),
504 'playlistId': playlist_id,
505 'playlistSecretToken': token,
509 track_id = str_or_none(track.get('id'))
510 url = track.get('permalink_url')
514 url = self._API_V2_BASE + 'tracks/' + track_id
516 url += '?secret_token=' + token
517 entries.append(self.url_result(
518 url, SoundcloudIE.ie_key(), track_id))
519 return self.playlist_result(
520 entries, playlist_id,
521 playlist.get('title'),
522 playlist.get('description'))
525 class SoundcloudSetIE(SoundcloudPlaylistBaseIE):
526 _VALID_URL = r'https?://(?:(?:www|m)\.)?soundcloud\.com/(?P<uploader>[\w\d-]+)/sets/(?P<slug_title>[\w\d-]+)(?:/(?P<token>[^?/]+))?'
527 IE_NAME = 'soundcloud:set'
529 'url': 'https://soundcloud.com/the-concept-band/sets/the-royal-concept-ep',
532 'title': 'The Royal Concept EP',
533 'description': 'md5:71d07087c7a449e8941a70a29e34671e',
535 'playlist_mincount': 5,
537 'url': 'https://soundcloud.com/the-concept-band/sets/the-royal-concept-ep/token',
538 'only_matching': True,
541 def _real_extract(self, url):
542 mobj = re.match(self._VALID_URL, url)
544 full_title = '%s/sets/%s' % mobj.group('uploader', 'slug_title')
545 token = mobj.group('token')
547 full_title += '/' + token
549 info = self._download_json(self._resolv_url(
550 self._BASE_URL + full_title), full_title)
553 msgs = (compat_str(err['error_message']) for err in info['errors'])
554 raise ExtractorError('unable to download video webpage: %s' % ','.join(msgs))
556 return self._extract_set(info, token)
559 class SoundcloudPagedPlaylistBaseIE(SoundcloudIE):
560 def _extract_playlist(self, base_url, playlist_id, playlist_title):
563 'linked_partitioning': '1',
566 query = COMMON_QUERY.copy()
572 for i in itertools.count():
573 response = self._download_json(
574 next_href, playlist_id,
575 'Downloading track page %s' % (i + 1), query=query)
577 collection = response['collection']
579 if not isinstance(collection, list):
582 # Empty collection may be returned, in this case we proceed
583 # straight to next_href
585 def resolve_entry(candidates):
586 for cand in candidates:
587 if not isinstance(cand, dict):
589 permalink_url = url_or_none(cand.get('permalink_url'))
590 if not permalink_url:
592 return self.url_result(
594 SoundcloudIE.ie_key() if SoundcloudIE.suitable(permalink_url) else None,
595 str_or_none(cand.get('id')), cand.get('title'))
598 entry = resolve_entry((e, e.get('track'), e.get('playlist')))
600 entries.append(entry)
602 next_href = response.get('next_href')
606 next_href = response['next_href']
607 parsed_next_href = compat_urlparse.urlparse(next_href)
608 query = compat_urlparse.parse_qs(parsed_next_href.query)
609 query.update(COMMON_QUERY)
614 'title': playlist_title,
619 class SoundcloudUserIE(SoundcloudPagedPlaylistBaseIE):
620 _VALID_URL = r'''(?x)
622 (?:(?:www|m)\.)?soundcloud\.com/
625 (?P<rsrc>tracks|albums|sets|reposts|likes|spotlight)
629 IE_NAME = 'soundcloud:user'
631 'url': 'https://soundcloud.com/soft-cell-official',
634 'title': 'Soft Cell (All)',
636 'playlist_mincount': 28,
638 'url': 'https://soundcloud.com/soft-cell-official/tracks',
641 'title': 'Soft Cell (Tracks)',
643 'playlist_mincount': 27,
645 'url': 'https://soundcloud.com/soft-cell-official/albums',
648 'title': 'Soft Cell (Albums)',
650 'playlist_mincount': 1,
652 'url': 'https://soundcloud.com/jcv246/sets',
655 'title': 'Jordi / cv (Sets)',
657 'playlist_mincount': 2,
659 'url': 'https://soundcloud.com/jcv246/reposts',
662 'title': 'Jordi / cv (Reposts)',
664 'playlist_mincount': 6,
666 'url': 'https://soundcloud.com/clalberg/likes',
669 'title': 'clalberg (Likes)',
671 'playlist_mincount': 5,
673 'url': 'https://soundcloud.com/grynpyret/spotlight',
676 'title': 'Grynpyret (Spotlight)',
678 'playlist_mincount': 1,
682 'all': 'stream/users/%s',
683 'tracks': 'users/%s/tracks',
684 'albums': 'users/%s/albums',
685 'sets': 'users/%s/playlists',
686 'reposts': 'stream/users/%s/reposts',
687 'likes': 'users/%s/likes',
688 'spotlight': 'users/%s/spotlight',
691 def _real_extract(self, url):
692 mobj = re.match(self._VALID_URL, url)
693 uploader = mobj.group('user')
695 user = self._download_json(
696 self._resolv_url(self._BASE_URL + uploader),
697 uploader, 'Downloading user info')
699 resource = mobj.group('rsrc') or 'all'
701 return self._extract_playlist(
702 self._API_V2_BASE + self._BASE_URL_MAP[resource] % user['id'],
703 str_or_none(user.get('id')),
704 '%s (%s)' % (user['username'], resource.capitalize()))
707 class SoundcloudTrackStationIE(SoundcloudPagedPlaylistBaseIE):
708 _VALID_URL = r'https?://(?:(?:www|m)\.)?soundcloud\.com/stations/track/[^/]+/(?P<id>[^/?#&]+)'
709 IE_NAME = 'soundcloud:trackstation'
711 'url': 'https://soundcloud.com/stations/track/officialsundial/your-text',
714 'title': 'Track station: your text',
716 'playlist_mincount': 47,
719 def _real_extract(self, url):
720 track_name = self._match_id(url)
722 track = self._download_json(self._resolv_url(url), track_name)
723 track_id = self._search_regex(
724 r'soundcloud:track-stations:(\d+)', track['id'], 'track id')
726 return self._extract_playlist(
727 self._API_V2_BASE + 'stations/%s/tracks' % track['id'],
728 track_id, 'Track station: %s' % track['title'])
731 class SoundcloudPlaylistIE(SoundcloudPlaylistBaseIE):
732 _VALID_URL = r'https?://api(?:-v2)?\.soundcloud\.com/playlists/(?P<id>[0-9]+)(?:/?\?secret_token=(?P<token>[^&]+?))?$'
733 IE_NAME = 'soundcloud:playlist'
735 'url': 'https://api.soundcloud.com/playlists/4110309',
738 'title': 'TILT Brass - Bowery Poetry Club, August \'03 [Non-Site SCR 02]',
739 'description': 're:.*?TILT Brass - Bowery Poetry Club',
744 def _real_extract(self, url):
745 mobj = re.match(self._VALID_URL, url)
746 playlist_id = mobj.group('id')
749 token = mobj.group('token')
751 query['secret_token'] = token
753 data = self._download_json(
754 self._API_V2_BASE + 'playlists/' + playlist_id,
755 playlist_id, 'Downloading playlist', query=query)
757 return self._extract_set(data, token)
760 class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
761 IE_NAME = 'soundcloud:search'
762 IE_DESC = 'Soundcloud search'
763 _MAX_RESULTS = float('inf')
765 'url': 'scsearch15:post-avant jazzcore',
767 'title': 'post-avant jazzcore',
769 'playlist_count': 15,
772 _SEARCH_KEY = 'scsearch'
773 _MAX_RESULTS_PER_PAGE = 200
774 _DEFAULT_RESULTS_PER_PAGE = 50
776 def _get_collection(self, endpoint, collection_id, **query):
778 query.get('limit', self._DEFAULT_RESULTS_PER_PAGE),
779 self._MAX_RESULTS_PER_PAGE)
782 'linked_partitioning': 1,
785 next_url = update_url_query(self._API_V2_BASE + endpoint, query)
787 collected_results = 0
789 for i in itertools.count(1):
790 response = self._download_json(
791 next_url, collection_id, 'Downloading page {0}'.format(i),
792 'Unable to download API page')
794 collection = response.get('collection', [])
798 collection = list(filter(bool, collection))
799 collected_results += len(collection)
801 for item in collection:
802 yield self.url_result(item['uri'], SoundcloudIE.ie_key())
804 if not collection or collected_results >= limit:
807 next_url = response.get('next_href')
811 def _get_n_results(self, query, n):
812 tracks = self._get_collection('search/tracks', query, limit=n, q=query)
813 return self.playlist_result(tracks, playlist_title=query)