]>
Commit | Line | Data |
---|---|---|
dcdb292f | 1 | # coding: utf-8 |
fbcd7b5f PH |
2 | from __future__ import unicode_literals |
3 | ||
aad0d6d5 | 4 | import re |
92790f4e | 5 | import itertools |
aad0d6d5 | 6 | |
2abf7cab | 7 | from .common import ( |
8 | InfoExtractor, | |
9 | SearchInfoExtractor | |
10 | ) | |
1cc79574 | 11 | from ..compat import ( |
aad0d6d5 | 12 | compat_str, |
668de34c | 13 | compat_urlparse, |
15707c7e | 14 | compat_urllib_parse_urlencode, |
1cc79574 PH |
15 | ) |
16 | from ..utils import ( | |
aad0d6d5 | 17 | ExtractorError, |
eb920777 | 18 | int_or_none, |
aad0d6d5 PH |
19 | unified_strdate, |
20 | ) | |
21 | ||
22 | ||
23 | class SoundcloudIE(InfoExtractor): | |
24 | """Information extractor for soundcloud.com | |
25 | To access the media, the uid of the song and a stream token | |
26 | must be extracted from the page source and the script must make | |
27 | a request to media.soundcloud.com/crossdomain.xml. Then | |
28 | the media can be grabbed by requesting from an url composed | |
29 | of the stream token and uid | |
30 | """ | |
31 | ||
20991253 | 32 | _VALID_URL = r'''(?x)^(?:https?://)? |
71507a11 | 33 | (?:(?:(?:www\.|m\.)?soundcloud\.com/ |
4ff50ef8 | 34 | (?P<uploader>[\w\d-]+)/ |
f7043ef3 | 35 | (?!(?:tracks|sets(?:/.+?)?|reposts|likes|spotlight)/?(?:$|[?#])) |
22a6f150 | 36 | (?P<title>[\w\d-]+)/? |
de2dd4c5 | 37 | (?P<token>[^?]+?)?(?:[?].*)?$) |
9296738f | 38 | |(?:api\.soundcloud\.com/tracks/(?P<track_id>\d+) |
0403b069 | 39 | (?:/?\?secret_token=(?P<secret_token>[^&]+))?) |
31c1cf5a | 40 | |(?P<player>(?:w|player|p.)\.soundcloud\.com/player/?.*?url=.*) |
eb6a41ba JMF |
41 | ) |
42 | ''' | |
fbcd7b5f | 43 | IE_NAME = 'soundcloud' |
12c167c8 JMF |
44 | _TESTS = [ |
45 | { | |
fbcd7b5f | 46 | 'url': 'http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy', |
fbcd7b5f PH |
47 | 'md5': 'ebef0a451b909710ed1d7787dddbf0d7', |
48 | 'info_dict': { | |
0eb9fb9f JMF |
49 | 'id': '62986583', |
50 | 'ext': 'mp3', | |
51 | 'upload_date': '20121011', | |
52 | 'description': 'No Downloads untill we record the finished version this weekend, i was too pumped n i had to post it , earl is prolly gonna b hella p.o\'d', | |
53 | 'uploader': 'E.T. ExTerrestrial Music', | |
54 | 'title': 'Lostin Powers - She so Heavy (SneakPreview) Adrian Ackers Blueprint 1', | |
55 | 'duration': 143, | |
4bfd294e | 56 | 'license': 'all-rights-reserved', |
12c167c8 JMF |
57 | } |
58 | }, | |
59 | # not streamable song | |
60 | { | |
fbcd7b5f PH |
61 | 'url': 'https://soundcloud.com/the-concept-band/goldrushed-mastered?in=the-concept-band/sets/the-royal-concept-ep', |
62 | 'info_dict': { | |
63 | 'id': '47127627', | |
64 | 'ext': 'mp3', | |
65 | 'title': 'Goldrushed', | |
63ad0315 | 66 | 'description': 'From Stockholm Sweden\r\nPovel / Magnus / Filip / David\r\nwww.theroyalconcept.com', |
fbcd7b5f PH |
67 | 'uploader': 'The Royal Concept', |
68 | 'upload_date': '20120521', | |
eb920777 | 69 | 'duration': 227, |
4bfd294e | 70 | 'license': 'all-rights-reserved', |
12c167c8 | 71 | }, |
fbcd7b5f | 72 | 'params': { |
12c167c8 | 73 | # rtmp |
fbcd7b5f | 74 | 'skip_download': True, |
12c167c8 JMF |
75 | }, |
76 | }, | |
de2dd4c5 JMF |
77 | # private link |
78 | { | |
fbcd7b5f PH |
79 | 'url': 'https://soundcloud.com/jaimemf/youtube-dl-test-video-a-y-baw/s-8Pjrp', |
80 | 'md5': 'aa0dd32bfea9b0c5ef4f02aacd080604', | |
81 | 'info_dict': { | |
82 | 'id': '123998367', | |
83 | 'ext': 'mp3', | |
84 | 'title': 'Youtube - Dl Test Video \'\' Ä↭', | |
85 | 'uploader': 'jaimeMF', | |
86 | 'description': 'test chars: \"\'/\\ä↭', | |
87 | 'upload_date': '20131209', | |
eb920777 | 88 | 'duration': 9, |
4bfd294e | 89 | 'license': 'all-rights-reserved', |
de2dd4c5 JMF |
90 | }, |
91 | }, | |
9296738f | 92 | # private link (alt format) |
93 | { | |
94 | 'url': 'https://api.soundcloud.com/tracks/123998367?secret_token=s-8Pjrp', | |
95 | 'md5': 'aa0dd32bfea9b0c5ef4f02aacd080604', | |
96 | 'info_dict': { | |
97 | 'id': '123998367', | |
98 | 'ext': 'mp3', | |
99 | 'title': 'Youtube - Dl Test Video \'\' Ä↭', | |
100 | 'uploader': 'jaimeMF', | |
101 | 'description': 'test chars: \"\'/\\ä↭', | |
102 | 'upload_date': '20131209', | |
103 | 'duration': 9, | |
4bfd294e | 104 | 'license': 'all-rights-reserved', |
9296738f | 105 | }, |
106 | }, | |
f67ca84d JMF |
107 | # downloadable song |
108 | { | |
00a82ea8 | 109 | 'url': 'https://soundcloud.com/oddsamples/bus-brakes', |
eae12e3f | 110 | 'md5': '7624f2351f8a3b2e7cd51522496e7631', |
fbcd7b5f | 111 | 'info_dict': { |
00a82ea8 | 112 | 'id': '128590877', |
eae12e3f | 113 | 'ext': 'mp3', |
00a82ea8 | 114 | 'title': 'Bus Brakes', |
0eb9fb9f | 115 | 'description': 'md5:0053ca6396e8d2fd7b7e1595ef12ab66', |
00a82ea8 S |
116 | 'uploader': 'oddsamples', |
117 | 'upload_date': '20140109', | |
118 | 'duration': 17, | |
4bfd294e | 119 | 'license': 'cc-by-sa', |
f67ca84d JMF |
120 | }, |
121 | }, | |
12c167c8 | 122 | ] |
aad0d6d5 | 123 | |
4c4765db | 124 | _CLIENT_ID = 'fDoItMDbsbZz8dY16ZzARCZmzgHBPotA' |
64bb5187 | 125 | _IPHONE_CLIENT_ID = '376f225bf427445fc4bfb6b99b72e0bf' |
7d239269 | 126 | |
fbdf8d15 S |
127 | @staticmethod |
128 | def _extract_urls(webpage): | |
129 | return [m.group('url') for m in re.finditer( | |
130 | r'<iframe[^>]+src=(["\'])(?P<url>(?:https?://)?(?:w\.)?soundcloud\.com/player.+?)\1', | |
131 | webpage)] | |
132 | ||
aad0d6d5 PH |
133 | def report_resolve(self, video_id): |
134 | """Report information extraction.""" | |
83622b6d | 135 | self.to_screen('%s: Resolving id' % video_id) |
aad0d6d5 | 136 | |
7d239269 JMF |
137 | @classmethod |
138 | def _resolv_url(cls, url): | |
139 | return 'http://api.soundcloud.com/resolve.json?url=' + url + '&client_id=' + cls._CLIENT_ID | |
140 | ||
de2dd4c5 | 141 | def _extract_info_dict(self, info, full_title=None, quiet=False, secret_token=None): |
12c167c8 JMF |
142 | track_id = compat_str(info['id']) |
143 | name = full_title or track_id | |
2a15e706 | 144 | if quiet: |
92790f4e | 145 | self.report_extraction(name) |
f62a77b9 S |
146 | thumbnail = info.get('artwork_url') |
147 | if isinstance(thumbnail, compat_str): | |
7d239269 | 148 | thumbnail = thumbnail.replace('-large', '-t500x500') |
fbcd7b5f | 149 | ext = 'mp3' |
12c167c8 | 150 | result = { |
2a15e706 | 151 | 'id': track_id, |
f62a77b9 S |
152 | 'uploader': info.get('user', {}).get('username'), |
153 | 'upload_date': unified_strdate(info.get('created_at')), | |
2a15e706 | 154 | 'title': info['title'], |
f62a77b9 | 155 | 'description': info.get('description'), |
7d239269 | 156 | 'thumbnail': thumbnail, |
eb920777 | 157 | 'duration': int_or_none(info.get('duration'), 1000), |
579657ad | 158 | 'webpage_url': info.get('permalink_url'), |
f62a77b9 | 159 | 'license': info.get('license'), |
7d239269 | 160 | } |
5e114e4b | 161 | formats = [] |
12c167c8 | 162 | if info.get('downloadable', False): |
64bb5187 | 163 | # We can build a direct link to the song |
2a15e706 | 164 | format_url = ( |
fbcd7b5f | 165 | 'https://api.soundcloud.com/tracks/{0}/download?client_id={1}'.format( |
2a15e706 | 166 | track_id, self._CLIENT_ID)) |
5e114e4b | 167 | formats.append({ |
2a15e706 | 168 | 'format_id': 'download', |
fbcd7b5f | 169 | 'ext': info.get('original_format', 'mp3'), |
2a15e706 | 170 | 'url': format_url, |
fb04e403 | 171 | 'vcodec': 'none', |
5e114e4b PH |
172 | 'preference': 10, |
173 | }) | |
174 | ||
175 | # We have to retrieve the url | |
20991253 | 176 | format_dict = self._download_json( |
489ffc11 S |
177 | 'http://api.soundcloud.com/i1/tracks/%s/streams' % track_id, |
178 | track_id, 'Downloading track url', query={ | |
179 | 'client_id': self._CLIENT_ID, | |
180 | 'secret_token': secret_token, | |
181 | }) | |
5e114e4b | 182 | |
5e114e4b | 183 | for key, stream_url in format_dict.items(): |
3a194cb4 S |
184 | abr = int_or_none(self._search_regex( |
185 | r'_(\d+)_url', key, 'audio bitrate', default=None)) | |
5e114e4b | 186 | if key.startswith('http'): |
3a194cb4 | 187 | stream_formats = [{ |
5e114e4b PH |
188 | 'format_id': key, |
189 | 'ext': ext, | |
190 | 'url': stream_url, | |
3a194cb4 | 191 | }] |
5e114e4b PH |
192 | elif key.startswith('rtmp'): |
193 | # The url doesn't have an rtmp app, we have to extract the playpath | |
194 | url, path = stream_url.split('mp3:', 1) | |
3a194cb4 | 195 | stream_formats = [{ |
5e114e4b PH |
196 | 'format_id': key, |
197 | 'url': url, | |
198 | 'play_path': 'mp3:' + path, | |
295df4ed | 199 | 'ext': 'flv', |
3a194cb4 | 200 | }] |
9b734718 | 201 | elif key.startswith('hls'): |
3a194cb4 | 202 | stream_formats = self._extract_m3u8_formats( |
9b734718 S |
203 | stream_url, track_id, 'mp3', entry_protocol='m3u8_native', |
204 | m3u8_id=key, fatal=False) | |
3a194cb4 S |
205 | else: |
206 | continue | |
207 | ||
208 | for f in stream_formats: | |
209 | f['abr'] = abr | |
210 | ||
211 | formats.extend(stream_formats) | |
212 | ||
213 | if not formats: | |
214 | # We fallback to the stream_url in the original info, this | |
215 | # cannot be always used, sometimes it can give an HTTP 404 error | |
216 | formats.append({ | |
217 | 'format_id': 'fallback', | |
218 | 'url': info['stream_url'] + '?client_id=' + self._CLIENT_ID, | |
219 | 'ext': ext, | |
220 | }) | |
221 | ||
222 | for f in formats: | |
223 | f['vcodec'] = 'none' | |
2a15e706 | 224 | |
562ceab1 S |
225 | self._check_formats(formats, track_id) |
226 | self._sort_formats(formats) | |
227 | result['formats'] = formats | |
64bb5187 | 228 | |
12c167c8 | 229 | return result |
7d239269 | 230 | |
aad0d6d5 | 231 | def _real_extract(self, url): |
eb6a41ba | 232 | mobj = re.match(self._VALID_URL, url, flags=re.VERBOSE) |
aad0d6d5 | 233 | if mobj is None: |
83622b6d | 234 | raise ExtractorError('Invalid URL: %s' % url) |
aad0d6d5 | 235 | |
eb6a41ba | 236 | track_id = mobj.group('track_id') |
4bfd294e | 237 | |
eb6a41ba JMF |
238 | if track_id is not None: |
239 | info_json_url = 'http://api.soundcloud.com/tracks/' + track_id + '.json?client_id=' + self._CLIENT_ID | |
240 | full_title = track_id | |
9296738f | 241 | token = mobj.group('secret_token') |
242 | if token: | |
611c1dd9 | 243 | info_json_url += '&secret_token=' + token |
31c1cf5a | 244 | elif mobj.group('player'): |
668de34c | 245 | query = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query) |
024ebb27 JMF |
246 | real_url = query['url'][0] |
247 | # If the token is in the query of the original url we have to | |
248 | # manually add it | |
249 | if 'secret_token' in query: | |
250 | real_url += '?secret_token=' + query['secret_token'][0] | |
251 | return self.url_result(real_url) | |
eb6a41ba JMF |
252 | else: |
253 | # extract uploader (which is in the url) | |
de2dd4c5 | 254 | uploader = mobj.group('uploader') |
eb6a41ba | 255 | # extract simple title (uploader + slug of song title) |
8bcc8756 | 256 | slug_title = mobj.group('title') |
de2dd4c5 JMF |
257 | token = mobj.group('token') |
258 | full_title = resolve_title = '%s/%s' % (uploader, slug_title) | |
259 | if token: | |
260 | resolve_title += '/%s' % token | |
5f6a1245 | 261 | |
eb6a41ba | 262 | self.report_resolve(full_title) |
5f6a1245 | 263 | |
de2dd4c5 | 264 | url = 'http://soundcloud.com/%s' % resolve_title |
eb6a41ba | 265 | info_json_url = self._resolv_url(url) |
20991253 | 266 | info = self._download_json(info_json_url, full_title, 'Downloading info JSON') |
aad0d6d5 | 267 | |
de2dd4c5 | 268 | return self._extract_info_dict(info, full_title, secret_token=token) |
aad0d6d5 | 269 | |
20991253 | 270 | |
7518a61d | 271 | class SoundcloudPlaylistBaseIE(SoundcloudIE): |
8e45e1cc S |
272 | @staticmethod |
273 | def _extract_id(e): | |
274 | return compat_str(e['id']) if e.get('id') else None | |
275 | ||
276 | def _extract_track_entries(self, tracks): | |
277 | return [ | |
278 | self.url_result( | |
279 | track['permalink_url'], SoundcloudIE.ie_key(), | |
280 | video_id=self._extract_id(track)) | |
281 | for track in tracks if track.get('permalink_url')] | |
282 | ||
283 | ||
7518a61d | 284 | class SoundcloudSetIE(SoundcloudPlaylistBaseIE): |
c808ef81 | 285 | _VALID_URL = r'https?://(?:(?:www|m)\.)?soundcloud\.com/(?P<uploader>[\w\d-]+)/sets/(?P<slug_title>[\w\d-]+)(?:/(?P<token>[^?/]+))?' |
fbcd7b5f | 286 | IE_NAME = 'soundcloud:set' |
22a6f150 PH |
287 | _TESTS = [{ |
288 | 'url': 'https://soundcloud.com/the-concept-band/sets/the-royal-concept-ep', | |
289 | 'info_dict': { | |
a9551e90 | 290 | 'id': '2284613', |
22a6f150 PH |
291 | 'title': 'The Royal Concept EP', |
292 | }, | |
293 | 'playlist_mincount': 6, | |
f7043ef3 S |
294 | }, { |
295 | 'url': 'https://soundcloud.com/the-concept-band/sets/the-royal-concept-ep/token', | |
296 | 'only_matching': True, | |
22a6f150 | 297 | }] |
aad0d6d5 | 298 | |
aad0d6d5 PH |
299 | def _real_extract(self, url): |
300 | mobj = re.match(self._VALID_URL, url) | |
aad0d6d5 PH |
301 | |
302 | # extract uploader (which is in the url) | |
2f834e93 | 303 | uploader = mobj.group('uploader') |
aad0d6d5 | 304 | # extract simple title (uploader + slug of song title) |
2f834e93 | 305 | slug_title = mobj.group('slug_title') |
aad0d6d5 | 306 | full_title = '%s/sets/%s' % (uploader, slug_title) |
2f834e93 | 307 | url = 'http://soundcloud.com/%s/sets/%s' % (uploader, slug_title) |
308 | ||
309 | token = mobj.group('token') | |
310 | if token: | |
311 | full_title += '/' + token | |
312 | url += '/' + token | |
aad0d6d5 PH |
313 | |
314 | self.report_resolve(full_title) | |
315 | ||
7d239269 | 316 | resolv_url = self._resolv_url(url) |
20991253 | 317 | info = self._download_json(resolv_url, full_title) |
aad0d6d5 | 318 | |
aad0d6d5 | 319 | if 'errors' in info: |
214e74bf JMF |
320 | msgs = (compat_str(err['error_message']) for err in info['errors']) |
321 | raise ExtractorError('unable to download video webpage: %s' % ','.join(msgs)) | |
aad0d6d5 | 322 | |
8e45e1cc | 323 | entries = self._extract_track_entries(info['tracks']) |
b14fa8e6 | 324 | |
22a6f150 PH |
325 | return { |
326 | '_type': 'playlist', | |
b14fa8e6 | 327 | 'entries': entries, |
a9551e90 | 328 | 'id': '%s' % info['id'], |
22a6f150 PH |
329 | 'title': info['title'], |
330 | } | |
92790f4e JMF |
331 | |
332 | ||
7518a61d | 333 | class SoundcloudUserIE(SoundcloudPlaylistBaseIE): |
16a08978 S |
334 | _VALID_URL = r'''(?x) |
335 | https?:// | |
336 | (?:(?:www|m)\.)?soundcloud\.com/ | |
337 | (?P<user>[^/]+) | |
338 | (?:/ | |
339 | (?P<rsrc>tracks|sets|reposts|likes|spotlight) | |
340 | )? | |
341 | /?(?:[?#].*)?$ | |
342 | ''' | |
fbcd7b5f | 343 | IE_NAME = 'soundcloud:user' |
22a6f150 | 344 | _TESTS = [{ |
80fb6d4a | 345 | 'url': 'https://soundcloud.com/the-akashic-chronicler', |
22a6f150 | 346 | 'info_dict': { |
80fb6d4a S |
347 | 'id': '114582580', |
348 | 'title': 'The Akashic Chronicler (All)', | |
22a6f150 | 349 | }, |
8e45e1cc | 350 | 'playlist_mincount': 74, |
22a6f150 | 351 | }, { |
80fb6d4a | 352 | 'url': 'https://soundcloud.com/the-akashic-chronicler/tracks', |
22a6f150 | 353 | 'info_dict': { |
80fb6d4a S |
354 | 'id': '114582580', |
355 | 'title': 'The Akashic Chronicler (Tracks)', | |
22a6f150 | 356 | }, |
8e45e1cc | 357 | 'playlist_mincount': 37, |
03b9c944 | 358 | }, { |
80fb6d4a S |
359 | 'url': 'https://soundcloud.com/the-akashic-chronicler/sets', |
360 | 'info_dict': { | |
361 | 'id': '114582580', | |
362 | 'title': 'The Akashic Chronicler (Playlists)', | |
363 | }, | |
8e45e1cc | 364 | 'playlist_mincount': 2, |
80fb6d4a S |
365 | }, { |
366 | 'url': 'https://soundcloud.com/the-akashic-chronicler/reposts', | |
367 | 'info_dict': { | |
368 | 'id': '114582580', | |
369 | 'title': 'The Akashic Chronicler (Reposts)', | |
370 | }, | |
66ce9702 | 371 | 'playlist_mincount': 7, |
80fb6d4a S |
372 | }, { |
373 | 'url': 'https://soundcloud.com/the-akashic-chronicler/likes', | |
374 | 'info_dict': { | |
375 | 'id': '114582580', | |
376 | 'title': 'The Akashic Chronicler (Likes)', | |
377 | }, | |
66ce9702 | 378 | 'playlist_mincount': 321, |
80fb6d4a S |
379 | }, { |
380 | 'url': 'https://soundcloud.com/grynpyret/spotlight', | |
381 | 'info_dict': { | |
382 | 'id': '7098329', | |
8e45e1cc | 383 | 'title': 'GRYNPYRET (Spotlight)', |
80fb6d4a S |
384 | }, |
385 | 'playlist_mincount': 1, | |
22a6f150 | 386 | }] |
92790f4e | 387 | |
80fb6d4a S |
388 | _API_BASE = 'https://api.soundcloud.com' |
389 | _API_V2_BASE = 'https://api-v2.soundcloud.com' | |
390 | ||
391 | _BASE_URL_MAP = { | |
392 | 'all': '%s/profile/soundcloud:users:%%s' % _API_V2_BASE, | |
393 | 'tracks': '%s/users/%%s/tracks' % _API_BASE, | |
394 | 'sets': '%s/users/%%s/playlists' % _API_V2_BASE, | |
395 | 'reposts': '%s/profile/soundcloud:users:%%s/reposts' % _API_V2_BASE, | |
396 | 'likes': '%s/users/%%s/likes' % _API_V2_BASE, | |
397 | 'spotlight': '%s/users/%%s/spotlight' % _API_V2_BASE, | |
398 | } | |
399 | ||
400 | _TITLE_MAP = { | |
401 | 'all': 'All', | |
402 | 'tracks': 'Tracks', | |
403 | 'sets': 'Playlists', | |
404 | 'reposts': 'Reposts', | |
405 | 'likes': 'Likes', | |
406 | 'spotlight': 'Spotlight', | |
407 | } | |
408 | ||
92790f4e JMF |
409 | def _real_extract(self, url): |
410 | mobj = re.match(self._VALID_URL, url) | |
411 | uploader = mobj.group('user') | |
412 | ||
413 | url = 'http://soundcloud.com/%s/' % uploader | |
414 | resolv_url = self._resolv_url(url) | |
20991253 PH |
415 | user = self._download_json( |
416 | resolv_url, uploader, 'Downloading user info') | |
80fb6d4a S |
417 | |
418 | resource = mobj.group('rsrc') or 'all' | |
419 | base_url = self._BASE_URL_MAP[resource] % user['id'] | |
420 | ||
97afd99a S |
421 | COMMON_QUERY = { |
422 | 'limit': 50, | |
423 | 'client_id': self._CLIENT_ID, | |
424 | 'linked_partitioning': '1', | |
425 | } | |
426 | ||
427 | query = COMMON_QUERY.copy() | |
428 | query['offset'] = 0 | |
429 | ||
15707c7e | 430 | next_href = base_url + '?' + compat_urllib_parse_urlencode(query) |
92790f4e | 431 | |
20991253 | 432 | entries = [] |
92790f4e | 433 | for i in itertools.count(): |
80fb6d4a S |
434 | response = self._download_json( |
435 | next_href, uploader, 'Downloading track page %s' % (i + 1)) | |
436 | ||
437 | collection = response['collection'] | |
80fb6d4a | 438 | if not collection: |
92790f4e | 439 | break |
80fb6d4a S |
440 | |
441 | def resolve_permalink_url(candidates): | |
442 | for cand in candidates: | |
443 | if isinstance(cand, dict): | |
444 | permalink_url = cand.get('permalink_url') | |
8e45e1cc | 445 | entry_id = self._extract_id(cand) |
80fb6d4a | 446 | if permalink_url and permalink_url.startswith('http'): |
8e45e1cc | 447 | return permalink_url, entry_id |
80fb6d4a S |
448 | |
449 | for e in collection: | |
8e45e1cc | 450 | permalink_url, entry_id = resolve_permalink_url((e, e.get('track'), e.get('playlist'))) |
80fb6d4a | 451 | if permalink_url: |
8e45e1cc | 452 | entries.append(self.url_result(permalink_url, video_id=entry_id)) |
80fb6d4a | 453 | |
97afd99a S |
454 | next_href = response.get('next_href') |
455 | if not next_href: | |
456 | break | |
457 | ||
458 | parsed_next_href = compat_urlparse.urlparse(response['next_href']) | |
459 | qs = compat_urlparse.parse_qs(parsed_next_href.query) | |
460 | qs.update(COMMON_QUERY) | |
461 | next_href = compat_urlparse.urlunparse( | |
15707c7e | 462 | parsed_next_href._replace(query=compat_urllib_parse_urlencode(qs, True))) |
92790f4e JMF |
463 | |
464 | return { | |
465 | '_type': 'playlist', | |
466 | 'id': compat_str(user['id']), | |
80fb6d4a | 467 | 'title': '%s (%s)' % (user['username'], self._TITLE_MAP[resource]), |
20991253 PH |
468 | 'entries': entries, |
469 | } | |
470 | ||
471 | ||
7518a61d | 472 | class SoundcloudPlaylistIE(SoundcloudPlaylistBaseIE): |
46f74bcf | 473 | _VALID_URL = r'https?://api\.soundcloud\.com/playlists/(?P<id>[0-9]+)(?:/?\?secret_token=(?P<token>[^&]+?))?$' |
20991253 | 474 | IE_NAME = 'soundcloud:playlist' |
46f74bcf PH |
475 | _TESTS = [{ |
476 | 'url': 'http://api.soundcloud.com/playlists/4110309', | |
477 | 'info_dict': { | |
478 | 'id': '4110309', | |
479 | 'title': 'TILT Brass - Bowery Poetry Club, August \'03 [Non-Site SCR 02]', | |
480 | 'description': 're:.*?TILT Brass - Bowery Poetry Club', | |
481 | }, | |
482 | 'playlist_count': 6, | |
483 | }] | |
20991253 PH |
484 | |
485 | def _real_extract(self, url): | |
486 | mobj = re.match(self._VALID_URL, url) | |
487 | playlist_id = mobj.group('id') | |
488 | base_url = '%s//api.soundcloud.com/playlists/%s.json?' % (self.http_scheme(), playlist_id) | |
489 | ||
2f834e93 | 490 | data_dict = { |
20991253 | 491 | 'client_id': self._CLIENT_ID, |
2f834e93 | 492 | } |
493 | token = mobj.group('token') | |
494 | ||
495 | if token: | |
496 | data_dict['secret_token'] = token | |
497 | ||
15707c7e | 498 | data = compat_urllib_parse_urlencode(data_dict) |
20991253 PH |
499 | data = self._download_json( |
500 | base_url + data, playlist_id, 'Downloading playlist') | |
501 | ||
8e45e1cc | 502 | entries = self._extract_track_entries(data['tracks']) |
20991253 PH |
503 | |
504 | return { | |
505 | '_type': 'playlist', | |
506 | 'id': playlist_id, | |
507 | 'title': data.get('title'), | |
508 | 'description': data.get('description'), | |
509 | 'entries': entries, | |
92790f4e | 510 | } |
2abf7cab | 511 | |
512 | ||
513 | class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE): | |
514 | IE_NAME = 'soundcloud:search' | |
515 | IE_DESC = 'Soundcloud search' | |
328a22e1 | 516 | _MAX_RESULTS = float('inf') |
2abf7cab | 517 | _TESTS = [{ |
518 | 'url': 'scsearch15:post-avant jazzcore', | |
519 | 'info_dict': { | |
520 | 'title': 'post-avant jazzcore', | |
521 | }, | |
522 | 'playlist_count': 15, | |
523 | }] | |
524 | ||
525 | _SEARCH_KEY = 'scsearch' | |
328a22e1 | 526 | _MAX_RESULTS_PER_PAGE = 200 |
527 | _DEFAULT_RESULTS_PER_PAGE = 50 | |
c30943b1 | 528 | _API_V2_BASE = 'https://api-v2.soundcloud.com' |
2abf7cab | 529 | |
530 | def _get_collection(self, endpoint, collection_id, **query): | |
a3372437 | 531 | limit = min( |
328a22e1 | 532 | query.get('limit', self._DEFAULT_RESULTS_PER_PAGE), |
533 | self._MAX_RESULTS_PER_PAGE) | |
f6c903e7 | 534 | query['limit'] = limit |
2abf7cab | 535 | query['client_id'] = self._CLIENT_ID |
536 | query['linked_partitioning'] = '1' | |
f6c903e7 | 537 | query['offset'] = 0 |
15707c7e | 538 | data = compat_urllib_parse_urlencode(query) |
f6c903e7 | 539 | next_url = '{0}{1}?{2}'.format(self._API_V2_BASE, endpoint, data) |
2abf7cab | 540 | |
2abf7cab | 541 | collected_results = 0 |
542 | ||
f6c903e7 | 543 | for i in itertools.count(1): |
7e347275 | 544 | response = self._download_json( |
f6c903e7 | 545 | next_url, collection_id, 'Downloading page {0}'.format(i), |
7e347275 | 546 | 'Unable to download API page') |
2abf7cab | 547 | |
f6c903e7 S |
548 | collection = response.get('collection', []) |
549 | if not collection: | |
550 | break | |
2abf7cab | 551 | |
f6c903e7 | 552 | collection = list(filter(bool, collection)) |
2abf7cab | 553 | collected_results += len(collection) |
554 | ||
f6c903e7 S |
555 | for item in collection: |
556 | yield self.url_result(item['uri'], SoundcloudIE.ie_key()) | |
2abf7cab | 557 | |
f6c903e7 | 558 | if not collection or collected_results >= limit: |
2abf7cab | 559 | break |
560 | ||
7e347275 | 561 | next_url = response.get('next_href') |
f6c903e7 S |
562 | if not next_url: |
563 | break | |
2abf7cab | 564 | |
565 | def _get_n_results(self, query, n): | |
f6c903e7 S |
566 | tracks = self._get_collection('/search/tracks', query, limit=n, q=query) |
567 | return self.playlist_result(tracks, playlist_title=query) |