]>
Commit | Line | Data |
---|---|---|
de2dd4c5 | 1 | # encoding: utf-8 |
fbcd7b5f PH |
2 | from __future__ import unicode_literals |
3 | ||
aad0d6d5 | 4 | import re |
92790f4e | 5 | import itertools |
aad0d6d5 PH |
6 | |
7 | from .common import InfoExtractor | |
1cc79574 | 8 | from ..compat import ( |
aad0d6d5 | 9 | compat_str, |
668de34c | 10 | compat_urlparse, |
92790f4e | 11 | compat_urllib_parse, |
1cc79574 PH |
12 | ) |
13 | from ..utils import ( | |
aad0d6d5 | 14 | ExtractorError, |
eb920777 | 15 | int_or_none, |
aad0d6d5 PH |
16 | unified_strdate, |
17 | ) | |
18 | ||
19 | ||
20 | class SoundcloudIE(InfoExtractor): | |
21 | """Information extractor for soundcloud.com | |
22 | To access the media, the uid of the song and a stream token | |
23 | must be extracted from the page source and the script must make | |
24 | a request to media.soundcloud.com/crossdomain.xml. Then | |
25 | the media can be grabbed by requesting from an url composed | |
26 | of the stream token and uid | |
27 | """ | |
28 | ||
20991253 | 29 | _VALID_URL = r'''(?x)^(?:https?://)? |
71507a11 | 30 | (?:(?:(?:www\.|m\.)?soundcloud\.com/ |
4ff50ef8 | 31 | (?P<uploader>[\w\d-]+)/ |
a219d175 | 32 | (?!sets/|(?:likes|tracks)/?(?:$|[?#])) |
22a6f150 | 33 | (?P<title>[\w\d-]+)/? |
de2dd4c5 | 34 | (?P<token>[^?]+?)?(?:[?].*)?$) |
9296738f | 35 | |(?:api\.soundcloud\.com/tracks/(?P<track_id>\d+) |
0403b069 | 36 | (?:/?\?secret_token=(?P<secret_token>[^&]+))?) |
31c1cf5a | 37 | |(?P<player>(?:w|player|p.)\.soundcloud\.com/player/?.*?url=.*) |
eb6a41ba JMF |
38 | ) |
39 | ''' | |
fbcd7b5f | 40 | IE_NAME = 'soundcloud' |
12c167c8 JMF |
41 | _TESTS = [ |
42 | { | |
fbcd7b5f | 43 | 'url': 'http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy', |
fbcd7b5f PH |
44 | 'md5': 'ebef0a451b909710ed1d7787dddbf0d7', |
45 | 'info_dict': { | |
0eb9fb9f JMF |
46 | 'id': '62986583', |
47 | 'ext': 'mp3', | |
48 | 'upload_date': '20121011', | |
49 | 'description': 'No Downloads untill we record the finished version this weekend, i was too pumped n i had to post it , earl is prolly gonna b hella p.o\'d', | |
50 | 'uploader': 'E.T. ExTerrestrial Music', | |
51 | 'title': 'Lostin Powers - She so Heavy (SneakPreview) Adrian Ackers Blueprint 1', | |
52 | 'duration': 143, | |
12c167c8 JMF |
53 | } |
54 | }, | |
55 | # not streamable song | |
56 | { | |
fbcd7b5f PH |
57 | 'url': 'https://soundcloud.com/the-concept-band/goldrushed-mastered?in=the-concept-band/sets/the-royal-concept-ep', |
58 | 'info_dict': { | |
59 | 'id': '47127627', | |
60 | 'ext': 'mp3', | |
61 | 'title': 'Goldrushed', | |
63ad0315 | 62 | 'description': 'From Stockholm Sweden\r\nPovel / Magnus / Filip / David\r\nwww.theroyalconcept.com', |
fbcd7b5f PH |
63 | 'uploader': 'The Royal Concept', |
64 | 'upload_date': '20120521', | |
eb920777 | 65 | 'duration': 227, |
12c167c8 | 66 | }, |
fbcd7b5f | 67 | 'params': { |
12c167c8 | 68 | # rtmp |
fbcd7b5f | 69 | 'skip_download': True, |
12c167c8 JMF |
70 | }, |
71 | }, | |
de2dd4c5 JMF |
72 | # private link |
73 | { | |
fbcd7b5f PH |
74 | 'url': 'https://soundcloud.com/jaimemf/youtube-dl-test-video-a-y-baw/s-8Pjrp', |
75 | 'md5': 'aa0dd32bfea9b0c5ef4f02aacd080604', | |
76 | 'info_dict': { | |
77 | 'id': '123998367', | |
78 | 'ext': 'mp3', | |
79 | 'title': 'Youtube - Dl Test Video \'\' Ä↭', | |
80 | 'uploader': 'jaimeMF', | |
81 | 'description': 'test chars: \"\'/\\ä↭', | |
82 | 'upload_date': '20131209', | |
eb920777 | 83 | 'duration': 9, |
de2dd4c5 JMF |
84 | }, |
85 | }, | |
9296738f | 86 | # private link (alt format) |
87 | { | |
88 | 'url': 'https://api.soundcloud.com/tracks/123998367?secret_token=s-8Pjrp', | |
89 | 'md5': 'aa0dd32bfea9b0c5ef4f02aacd080604', | |
90 | 'info_dict': { | |
91 | 'id': '123998367', | |
92 | 'ext': 'mp3', | |
93 | 'title': 'Youtube - Dl Test Video \'\' Ä↭', | |
94 | 'uploader': 'jaimeMF', | |
95 | 'description': 'test chars: \"\'/\\ä↭', | |
96 | 'upload_date': '20131209', | |
97 | 'duration': 9, | |
98 | }, | |
99 | }, | |
f67ca84d JMF |
100 | # downloadable song |
101 | { | |
00a82ea8 | 102 | 'url': 'https://soundcloud.com/oddsamples/bus-brakes', |
eae12e3f | 103 | 'md5': '7624f2351f8a3b2e7cd51522496e7631', |
fbcd7b5f | 104 | 'info_dict': { |
00a82ea8 | 105 | 'id': '128590877', |
eae12e3f | 106 | 'ext': 'mp3', |
00a82ea8 | 107 | 'title': 'Bus Brakes', |
0eb9fb9f | 108 | 'description': 'md5:0053ca6396e8d2fd7b7e1595ef12ab66', |
00a82ea8 S |
109 | 'uploader': 'oddsamples', |
110 | 'upload_date': '20140109', | |
111 | 'duration': 17, | |
f67ca84d JMF |
112 | }, |
113 | }, | |
12c167c8 | 114 | ] |
aad0d6d5 | 115 | |
7d239269 | 116 | _CLIENT_ID = 'b45b1aa10f1ac2941910a7f0d10f8e28' |
64bb5187 | 117 | _IPHONE_CLIENT_ID = '376f225bf427445fc4bfb6b99b72e0bf' |
7d239269 | 118 | |
aad0d6d5 PH |
119 | def report_resolve(self, video_id): |
120 | """Report information extraction.""" | |
83622b6d | 121 | self.to_screen('%s: Resolving id' % video_id) |
aad0d6d5 | 122 | |
7d239269 JMF |
123 | @classmethod |
124 | def _resolv_url(cls, url): | |
125 | return 'http://api.soundcloud.com/resolve.json?url=' + url + '&client_id=' + cls._CLIENT_ID | |
126 | ||
de2dd4c5 | 127 | def _extract_info_dict(self, info, full_title=None, quiet=False, secret_token=None): |
12c167c8 JMF |
128 | track_id = compat_str(info['id']) |
129 | name = full_title or track_id | |
2a15e706 | 130 | if quiet: |
92790f4e | 131 | self.report_extraction(name) |
7d239269 JMF |
132 | |
133 | thumbnail = info['artwork_url'] | |
134 | if thumbnail is not None: | |
135 | thumbnail = thumbnail.replace('-large', '-t500x500') | |
fbcd7b5f | 136 | ext = 'mp3' |
12c167c8 | 137 | result = { |
2a15e706 | 138 | 'id': track_id, |
7d239269 JMF |
139 | 'uploader': info['user']['username'], |
140 | 'upload_date': unified_strdate(info['created_at']), | |
2a15e706 | 141 | 'title': info['title'], |
7d239269 JMF |
142 | 'description': info['description'], |
143 | 'thumbnail': thumbnail, | |
eb920777 | 144 | 'duration': int_or_none(info.get('duration'), 1000), |
579657ad | 145 | 'webpage_url': info.get('permalink_url'), |
7d239269 | 146 | } |
5e114e4b | 147 | formats = [] |
12c167c8 | 148 | if info.get('downloadable', False): |
64bb5187 | 149 | # We can build a direct link to the song |
2a15e706 | 150 | format_url = ( |
fbcd7b5f | 151 | 'https://api.soundcloud.com/tracks/{0}/download?client_id={1}'.format( |
2a15e706 | 152 | track_id, self._CLIENT_ID)) |
5e114e4b | 153 | formats.append({ |
2a15e706 | 154 | 'format_id': 'download', |
fbcd7b5f | 155 | 'ext': info.get('original_format', 'mp3'), |
2a15e706 | 156 | 'url': format_url, |
fb04e403 | 157 | 'vcodec': 'none', |
5e114e4b PH |
158 | 'preference': 10, |
159 | }) | |
160 | ||
161 | # We have to retrieve the url | |
162 | streams_url = ('http://api.soundcloud.com/i1/tracks/{0}/streams?' | |
9e1a5b84 | 163 | 'client_id={1}&secret_token={2}'.format(track_id, self._IPHONE_CLIENT_ID, secret_token)) |
20991253 | 164 | format_dict = self._download_json( |
5e114e4b PH |
165 | streams_url, |
166 | track_id, 'Downloading track url') | |
167 | ||
5e114e4b PH |
168 | for key, stream_url in format_dict.items(): |
169 | if key.startswith('http'): | |
170 | formats.append({ | |
171 | 'format_id': key, | |
172 | 'ext': ext, | |
173 | 'url': stream_url, | |
174 | 'vcodec': 'none', | |
175 | }) | |
176 | elif key.startswith('rtmp'): | |
177 | # The url doesn't have an rtmp app, we have to extract the playpath | |
178 | url, path = stream_url.split('mp3:', 1) | |
179 | formats.append({ | |
180 | 'format_id': key, | |
181 | 'url': url, | |
182 | 'play_path': 'mp3:' + path, | |
295df4ed | 183 | 'ext': 'flv', |
5e114e4b PH |
184 | 'vcodec': 'none', |
185 | }) | |
2a15e706 PH |
186 | |
187 | if not formats: | |
64bb5187 JMF |
188 | # We fallback to the stream_url in the original info, this |
189 | # cannot be always used, sometimes it can give an HTTP 404 error | |
2a15e706 | 190 | formats.append({ |
fbcd7b5f | 191 | 'format_id': 'fallback', |
2a15e706 PH |
192 | 'url': info['stream_url'] + '?client_id=' + self._CLIENT_ID, |
193 | 'ext': ext, | |
fb04e403 | 194 | 'vcodec': 'none', |
2a15e706 PH |
195 | }) |
196 | ||
fbcd7b5f | 197 | for f in formats: |
2a15e706 | 198 | if f['format_id'].startswith('http'): |
fbcd7b5f | 199 | f['protocol'] = 'http' |
2a15e706 | 200 | if f['format_id'].startswith('rtmp'): |
fbcd7b5f | 201 | f['protocol'] = 'rtmp' |
2a15e706 | 202 | |
562ceab1 S |
203 | self._check_formats(formats, track_id) |
204 | self._sort_formats(formats) | |
205 | result['formats'] = formats | |
64bb5187 | 206 | |
12c167c8 | 207 | return result |
7d239269 | 208 | |
aad0d6d5 | 209 | def _real_extract(self, url): |
eb6a41ba | 210 | mobj = re.match(self._VALID_URL, url, flags=re.VERBOSE) |
aad0d6d5 | 211 | if mobj is None: |
83622b6d | 212 | raise ExtractorError('Invalid URL: %s' % url) |
aad0d6d5 | 213 | |
eb6a41ba | 214 | track_id = mobj.group('track_id') |
de2dd4c5 | 215 | token = None |
eb6a41ba JMF |
216 | if track_id is not None: |
217 | info_json_url = 'http://api.soundcloud.com/tracks/' + track_id + '.json?client_id=' + self._CLIENT_ID | |
218 | full_title = track_id | |
9296738f | 219 | token = mobj.group('secret_token') |
220 | if token: | |
221 | info_json_url += "&secret_token=" + token | |
31c1cf5a | 222 | elif mobj.group('player'): |
668de34c | 223 | query = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query) |
024ebb27 JMF |
224 | real_url = query['url'][0] |
225 | # If the token is in the query of the original url we have to | |
226 | # manually add it | |
227 | if 'secret_token' in query: | |
228 | real_url += '?secret_token=' + query['secret_token'][0] | |
229 | return self.url_result(real_url) | |
eb6a41ba JMF |
230 | else: |
231 | # extract uploader (which is in the url) | |
de2dd4c5 | 232 | uploader = mobj.group('uploader') |
eb6a41ba | 233 | # extract simple title (uploader + slug of song title) |
8bcc8756 | 234 | slug_title = mobj.group('title') |
de2dd4c5 JMF |
235 | token = mobj.group('token') |
236 | full_title = resolve_title = '%s/%s' % (uploader, slug_title) | |
237 | if token: | |
238 | resolve_title += '/%s' % token | |
5f6a1245 | 239 | |
eb6a41ba | 240 | self.report_resolve(full_title) |
5f6a1245 | 241 | |
de2dd4c5 | 242 | url = 'http://soundcloud.com/%s' % resolve_title |
eb6a41ba | 243 | info_json_url = self._resolv_url(url) |
20991253 | 244 | info = self._download_json(info_json_url, full_title, 'Downloading info JSON') |
aad0d6d5 | 245 | |
de2dd4c5 | 246 | return self._extract_info_dict(info, full_title, secret_token=token) |
aad0d6d5 | 247 | |
20991253 | 248 | |
7d239269 | 249 | class SoundcloudSetIE(SoundcloudIE): |
c808ef81 | 250 | _VALID_URL = r'https?://(?:(?:www|m)\.)?soundcloud\.com/(?P<uploader>[\w\d-]+)/sets/(?P<slug_title>[\w\d-]+)(?:/(?P<token>[^?/]+))?' |
fbcd7b5f | 251 | IE_NAME = 'soundcloud:set' |
22a6f150 PH |
252 | _TESTS = [{ |
253 | 'url': 'https://soundcloud.com/the-concept-band/sets/the-royal-concept-ep', | |
254 | 'info_dict': { | |
a9551e90 | 255 | 'id': '2284613', |
22a6f150 PH |
256 | 'title': 'The Royal Concept EP', |
257 | }, | |
258 | 'playlist_mincount': 6, | |
259 | }] | |
aad0d6d5 | 260 | |
aad0d6d5 PH |
261 | def _real_extract(self, url): |
262 | mobj = re.match(self._VALID_URL, url) | |
aad0d6d5 PH |
263 | |
264 | # extract uploader (which is in the url) | |
2f834e93 | 265 | uploader = mobj.group('uploader') |
aad0d6d5 | 266 | # extract simple title (uploader + slug of song title) |
2f834e93 | 267 | slug_title = mobj.group('slug_title') |
aad0d6d5 | 268 | full_title = '%s/sets/%s' % (uploader, slug_title) |
2f834e93 | 269 | url = 'http://soundcloud.com/%s/sets/%s' % (uploader, slug_title) |
270 | ||
271 | token = mobj.group('token') | |
272 | if token: | |
273 | full_title += '/' + token | |
274 | url += '/' + token | |
aad0d6d5 PH |
275 | |
276 | self.report_resolve(full_title) | |
277 | ||
7d239269 | 278 | resolv_url = self._resolv_url(url) |
20991253 | 279 | info = self._download_json(resolv_url, full_title) |
aad0d6d5 | 280 | |
aad0d6d5 | 281 | if 'errors' in info: |
214e74bf JMF |
282 | msgs = (compat_str(err['error_message']) for err in info['errors']) |
283 | raise ExtractorError('unable to download video webpage: %s' % ','.join(msgs)) | |
aad0d6d5 | 284 | |
22a6f150 PH |
285 | return { |
286 | '_type': 'playlist', | |
2f834e93 | 287 | 'entries': [self._extract_info_dict(track, secret_token=token) for track in info['tracks']], |
a9551e90 | 288 | 'id': '%s' % info['id'], |
22a6f150 PH |
289 | 'title': info['title'], |
290 | } | |
92790f4e JMF |
291 | |
292 | ||
293 | class SoundcloudUserIE(SoundcloudIE): | |
c808ef81 | 294 | _VALID_URL = r'https?://(?:(?:www|m)\.)?soundcloud\.com/(?P<user>[^/]+)/?((?P<rsrc>tracks|likes)/?)?(\?.*)?$' |
fbcd7b5f | 295 | IE_NAME = 'soundcloud:user' |
22a6f150 PH |
296 | _TESTS = [{ |
297 | 'url': 'https://soundcloud.com/the-concept-band', | |
298 | 'info_dict': { | |
299 | 'id': '9615865', | |
300 | 'title': 'The Royal Concept', | |
301 | }, | |
302 | 'playlist_mincount': 12 | |
303 | }, { | |
304 | 'url': 'https://soundcloud.com/the-concept-band/likes', | |
305 | 'info_dict': { | |
306 | 'id': '9615865', | |
307 | 'title': 'The Royal Concept', | |
308 | }, | |
309 | 'playlist_mincount': 1, | |
310 | }] | |
92790f4e JMF |
311 | |
312 | def _real_extract(self, url): | |
313 | mobj = re.match(self._VALID_URL, url) | |
314 | uploader = mobj.group('user') | |
3941669d | 315 | resource = mobj.group('rsrc') |
316 | if resource is None: | |
317 | resource = 'tracks' | |
318 | elif resource == 'likes': | |
319 | resource = 'favorites' | |
92790f4e JMF |
320 | |
321 | url = 'http://soundcloud.com/%s/' % uploader | |
322 | resolv_url = self._resolv_url(url) | |
20991253 PH |
323 | user = self._download_json( |
324 | resolv_url, uploader, 'Downloading user info') | |
3941669d | 325 | base_url = 'http://api.soundcloud.com/users/%s/%s.json?' % (uploader, resource) |
92790f4e | 326 | |
20991253 | 327 | entries = [] |
92790f4e | 328 | for i in itertools.count(): |
20991253 PH |
329 | data = compat_urllib_parse.urlencode({ |
330 | 'offset': i * 50, | |
3941669d | 331 | 'limit': 50, |
20991253 PH |
332 | 'client_id': self._CLIENT_ID, |
333 | }) | |
334 | new_entries = self._download_json( | |
335 | base_url + data, uploader, 'Downloading track page %s' % (i + 1)) | |
3941669d | 336 | if len(new_entries) == 0: |
337 | self.to_screen('%s: End page received' % uploader) | |
92790f4e | 338 | break |
1ae72fb2 | 339 | entries.extend(self.url_result(e['permalink_url'], 'Soundcloud') for e in new_entries) |
92790f4e JMF |
340 | |
341 | return { | |
342 | '_type': 'playlist', | |
343 | 'id': compat_str(user['id']), | |
344 | 'title': user['username'], | |
20991253 PH |
345 | 'entries': entries, |
346 | } | |
347 | ||
348 | ||
349 | class SoundcloudPlaylistIE(SoundcloudIE): | |
46f74bcf | 350 | _VALID_URL = r'https?://api\.soundcloud\.com/playlists/(?P<id>[0-9]+)(?:/?\?secret_token=(?P<token>[^&]+?))?$' |
20991253 | 351 | IE_NAME = 'soundcloud:playlist' |
46f74bcf PH |
352 | _TESTS = [{ |
353 | 'url': 'http://api.soundcloud.com/playlists/4110309', | |
354 | 'info_dict': { | |
355 | 'id': '4110309', | |
356 | 'title': 'TILT Brass - Bowery Poetry Club, August \'03 [Non-Site SCR 02]', | |
357 | 'description': 're:.*?TILT Brass - Bowery Poetry Club', | |
358 | }, | |
359 | 'playlist_count': 6, | |
360 | }] | |
20991253 PH |
361 | |
362 | def _real_extract(self, url): | |
363 | mobj = re.match(self._VALID_URL, url) | |
364 | playlist_id = mobj.group('id') | |
365 | base_url = '%s//api.soundcloud.com/playlists/%s.json?' % (self.http_scheme(), playlist_id) | |
366 | ||
2f834e93 | 367 | data_dict = { |
20991253 | 368 | 'client_id': self._CLIENT_ID, |
2f834e93 | 369 | } |
370 | token = mobj.group('token') | |
371 | ||
372 | if token: | |
373 | data_dict['secret_token'] = token | |
374 | ||
375 | data = compat_urllib_parse.urlencode(data_dict) | |
20991253 PH |
376 | data = self._download_json( |
377 | base_url + data, playlist_id, 'Downloading playlist') | |
378 | ||
379 | entries = [ | |
2f834e93 | 380 | self._extract_info_dict(t, quiet=True, secret_token=token) |
b74e86f4 | 381 | for t in data['tracks']] |
20991253 PH |
382 | |
383 | return { | |
384 | '_type': 'playlist', | |
385 | 'id': playlist_id, | |
386 | 'title': data.get('title'), | |
387 | 'description': data.get('description'), | |
388 | 'entries': entries, | |
92790f4e | 389 | } |