]>
Commit | Line | Data |
---|---|---|
1 | # encoding: utf-8 | |
2 | import json | |
3 | import re | |
4 | import itertools | |
5 | ||
6 | from .common import InfoExtractor | |
7 | from ..utils import ( | |
8 | compat_str, | |
9 | compat_urlparse, | |
10 | compat_urllib_parse, | |
11 | ||
12 | ExtractorError, | |
13 | unified_strdate, | |
14 | ) | |
15 | ||
16 | ||
17 | class SoundcloudIE(InfoExtractor): | |
18 | """Information extractor for soundcloud.com | |
19 | To access the media, the uid of the song and a stream token | |
20 | must be extracted from the page source and the script must make | |
21 | a request to media.soundcloud.com/crossdomain.xml. Then | |
22 | the media can be grabbed by requesting from an url composed | |
23 | of the stream token and uid | |
24 | """ | |
25 | ||
26 | _VALID_URL = r'''^(?:https?://)? | |
27 | (?:(?:(?:www\.)?soundcloud\.com/ | |
28 | (?P<uploader>[\w\d-]+)/(?P<title>[\w\d-]+)/? | |
29 | (?P<token>[^?]+?)?(?:[?].*)?$) | |
30 | |(?:api\.soundcloud\.com/tracks/(?P<track_id>\d+)) | |
31 | |(?P<widget>w\.soundcloud\.com/player/?.*?url=.*) | |
32 | ) | |
33 | ''' | |
34 | IE_NAME = u'soundcloud' | |
35 | _TESTS = [ | |
36 | { | |
37 | u'url': u'http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy', | |
38 | u'file': u'62986583.mp3', | |
39 | u'md5': u'ebef0a451b909710ed1d7787dddbf0d7', | |
40 | u'info_dict': { | |
41 | u"upload_date": u"20121011", | |
42 | u"description": u"No Downloads untill we record the finished version this weekend, i was too pumped n i had to post it , earl is prolly gonna b hella p.o'd", | |
43 | u"uploader": u"E.T. ExTerrestrial Music", | |
44 | u"title": u"Lostin Powers - She so Heavy (SneakPreview) Adrian Ackers Blueprint 1" | |
45 | } | |
46 | }, | |
47 | # not streamable song | |
48 | { | |
49 | u'url': u'https://soundcloud.com/the-concept-band/goldrushed-mastered?in=the-concept-band/sets/the-royal-concept-ep', | |
50 | u'info_dict': { | |
51 | u'id': u'47127627', | |
52 | u'ext': u'mp3', | |
53 | u'title': u'Goldrushed', | |
54 | u'uploader': u'The Royal Concept', | |
55 | u'upload_date': u'20120521', | |
56 | }, | |
57 | u'params': { | |
58 | # rtmp | |
59 | u'skip_download': True, | |
60 | }, | |
61 | }, | |
62 | # private link | |
63 | { | |
64 | u'url': u'https://soundcloud.com/jaimemf/youtube-dl-test-video-a-y-baw/s-8Pjrp', | |
65 | u'md5': u'aa0dd32bfea9b0c5ef4f02aacd080604', | |
66 | u'info_dict': { | |
67 | u'id': u'123998367', | |
68 | u'ext': u'mp3', | |
69 | u'title': u'Youtube - Dl Test Video \'\' Ä↭', | |
70 | u'uploader': u'jaimeMF', | |
71 | u'description': u'test chars: \"\'/\\ä↭', | |
72 | u'upload_date': u'20131209', | |
73 | }, | |
74 | }, | |
75 | ] | |
76 | ||
77 | _CLIENT_ID = 'b45b1aa10f1ac2941910a7f0d10f8e28' | |
78 | _IPHONE_CLIENT_ID = '376f225bf427445fc4bfb6b99b72e0bf' | |
79 | ||
80 | @classmethod | |
81 | def suitable(cls, url): | |
82 | return re.match(cls._VALID_URL, url, flags=re.VERBOSE) is not None | |
83 | ||
84 | def report_resolve(self, video_id): | |
85 | """Report information extraction.""" | |
86 | self.to_screen(u'%s: Resolving id' % video_id) | |
87 | ||
88 | @classmethod | |
89 | def _resolv_url(cls, url): | |
90 | return 'http://api.soundcloud.com/resolve.json?url=' + url + '&client_id=' + cls._CLIENT_ID | |
91 | ||
92 | def _extract_info_dict(self, info, full_title=None, quiet=False, secret_token=None): | |
93 | track_id = compat_str(info['id']) | |
94 | name = full_title or track_id | |
95 | if quiet: | |
96 | self.report_extraction(name) | |
97 | ||
98 | thumbnail = info['artwork_url'] | |
99 | if thumbnail is not None: | |
100 | thumbnail = thumbnail.replace('-large', '-t500x500') | |
101 | ext = info.get('original_format', u'mp3') | |
102 | result = { | |
103 | 'id': track_id, | |
104 | 'uploader': info['user']['username'], | |
105 | 'upload_date': unified_strdate(info['created_at']), | |
106 | 'title': info['title'], | |
107 | 'description': info['description'], | |
108 | 'thumbnail': thumbnail, | |
109 | } | |
110 | if info.get('downloadable', False): | |
111 | # We can build a direct link to the song | |
112 | format_url = ( | |
113 | u'https://api.soundcloud.com/tracks/{0}/download?client_id={1}'.format( | |
114 | track_id, self._CLIENT_ID)) | |
115 | result['formats'] = [{ | |
116 | 'format_id': 'download', | |
117 | 'ext': ext, | |
118 | 'url': format_url, | |
119 | 'vcodec': 'none', | |
120 | }] | |
121 | else: | |
122 | # We have to retrieve the url | |
123 | streams_url = ('http://api.soundcloud.com/i1/tracks/{0}/streams?' | |
124 | 'client_id={1}&secret_token={2}'.format(track_id, self._IPHONE_CLIENT_ID, secret_token)) | |
125 | stream_json = self._download_webpage( | |
126 | streams_url, | |
127 | track_id, u'Downloading track url') | |
128 | ||
129 | formats = [] | |
130 | format_dict = json.loads(stream_json) | |
131 | for key, stream_url in format_dict.items(): | |
132 | if key.startswith(u'http'): | |
133 | formats.append({ | |
134 | 'format_id': key, | |
135 | 'ext': ext, | |
136 | 'url': stream_url, | |
137 | 'vcodec': 'none', | |
138 | }) | |
139 | elif key.startswith(u'rtmp'): | |
140 | # The url doesn't have an rtmp app, we have to extract the playpath | |
141 | url, path = stream_url.split('mp3:', 1) | |
142 | formats.append({ | |
143 | 'format_id': key, | |
144 | 'url': url, | |
145 | 'play_path': 'mp3:' + path, | |
146 | 'ext': ext, | |
147 | 'vcodec': 'none', | |
148 | }) | |
149 | ||
150 | if not formats: | |
151 | # We fallback to the stream_url in the original info, this | |
152 | # cannot be always used, sometimes it can give an HTTP 404 error | |
153 | formats.append({ | |
154 | 'format_id': u'fallback', | |
155 | 'url': info['stream_url'] + '?client_id=' + self._CLIENT_ID, | |
156 | 'ext': ext, | |
157 | 'vcodec': 'none', | |
158 | }) | |
159 | ||
160 | def format_pref(f): | |
161 | if f['format_id'].startswith('http'): | |
162 | return 2 | |
163 | if f['format_id'].startswith('rtmp'): | |
164 | return 1 | |
165 | return 0 | |
166 | ||
167 | formats.sort(key=format_pref) | |
168 | result['formats'] = formats | |
169 | ||
170 | return result | |
171 | ||
172 | def _real_extract(self, url): | |
173 | mobj = re.match(self._VALID_URL, url, flags=re.VERBOSE) | |
174 | if mobj is None: | |
175 | raise ExtractorError(u'Invalid URL: %s' % url) | |
176 | ||
177 | track_id = mobj.group('track_id') | |
178 | token = None | |
179 | if track_id is not None: | |
180 | info_json_url = 'http://api.soundcloud.com/tracks/' + track_id + '.json?client_id=' + self._CLIENT_ID | |
181 | full_title = track_id | |
182 | elif mobj.group('widget'): | |
183 | query = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query) | |
184 | return self.url_result(query['url'][0], ie='Soundcloud') | |
185 | else: | |
186 | # extract uploader (which is in the url) | |
187 | uploader = mobj.group('uploader') | |
188 | # extract simple title (uploader + slug of song title) | |
189 | slug_title = mobj.group('title') | |
190 | token = mobj.group('token') | |
191 | full_title = resolve_title = '%s/%s' % (uploader, slug_title) | |
192 | if token: | |
193 | resolve_title += '/%s' % token | |
194 | ||
195 | self.report_resolve(full_title) | |
196 | ||
197 | url = 'http://soundcloud.com/%s' % resolve_title | |
198 | info_json_url = self._resolv_url(url) | |
199 | info_json = self._download_webpage(info_json_url, full_title, u'Downloading info JSON') | |
200 | ||
201 | info = json.loads(info_json) | |
202 | return self._extract_info_dict(info, full_title, secret_token=token) | |
203 | ||
204 | class SoundcloudSetIE(SoundcloudIE): | |
205 | _VALID_URL = r'^(?:https?://)?(?:www\.)?soundcloud\.com/([\w\d-]+)/sets/([\w\d-]+)(?:[?].*)?$' | |
206 | IE_NAME = u'soundcloud:set' | |
207 | # it's in tests/test_playlists.py | |
208 | _TESTS = [] | |
209 | ||
210 | def _real_extract(self, url): | |
211 | mobj = re.match(self._VALID_URL, url) | |
212 | if mobj is None: | |
213 | raise ExtractorError(u'Invalid URL: %s' % url) | |
214 | ||
215 | # extract uploader (which is in the url) | |
216 | uploader = mobj.group(1) | |
217 | # extract simple title (uploader + slug of song title) | |
218 | slug_title = mobj.group(2) | |
219 | full_title = '%s/sets/%s' % (uploader, slug_title) | |
220 | ||
221 | self.report_resolve(full_title) | |
222 | ||
223 | url = 'http://soundcloud.com/%s/sets/%s' % (uploader, slug_title) | |
224 | resolv_url = self._resolv_url(url) | |
225 | info_json = self._download_webpage(resolv_url, full_title) | |
226 | ||
227 | info = json.loads(info_json) | |
228 | if 'errors' in info: | |
229 | for err in info['errors']: | |
230 | self._downloader.report_error(u'unable to download video webpage: %s' % compat_str(err['error_message'])) | |
231 | return | |
232 | ||
233 | self.report_extraction(full_title) | |
234 | return {'_type': 'playlist', | |
235 | 'entries': [self._extract_info_dict(track) for track in info['tracks']], | |
236 | 'id': info['id'], | |
237 | 'title': info['title'], | |
238 | } | |
239 | ||
240 | ||
241 | class SoundcloudUserIE(SoundcloudIE): | |
242 | _VALID_URL = r'https?://(www\.)?soundcloud\.com/(?P<user>[^/]+)(/?(tracks/)?)?(\?.*)?$' | |
243 | IE_NAME = u'soundcloud:user' | |
244 | ||
245 | # it's in tests/test_playlists.py | |
246 | _TESTS = [] | |
247 | ||
248 | def _real_extract(self, url): | |
249 | mobj = re.match(self._VALID_URL, url) | |
250 | uploader = mobj.group('user') | |
251 | ||
252 | url = 'http://soundcloud.com/%s/' % uploader | |
253 | resolv_url = self._resolv_url(url) | |
254 | user_json = self._download_webpage(resolv_url, uploader, | |
255 | u'Downloading user info') | |
256 | user = json.loads(user_json) | |
257 | ||
258 | tracks = [] | |
259 | for i in itertools.count(): | |
260 | data = compat_urllib_parse.urlencode({'offset': i*50, | |
261 | 'client_id': self._CLIENT_ID, | |
262 | }) | |
263 | tracks_url = 'http://api.soundcloud.com/users/%s/tracks.json?' % user['id'] + data | |
264 | response = self._download_webpage(tracks_url, uploader, | |
265 | u'Downloading tracks page %s' % (i+1)) | |
266 | new_tracks = json.loads(response) | |
267 | tracks.extend(self._extract_info_dict(track, quiet=True) for track in new_tracks) | |
268 | if len(new_tracks) < 50: | |
269 | break | |
270 | ||
271 | return { | |
272 | '_type': 'playlist', | |
273 | 'id': compat_str(user['id']), | |
274 | 'title': user['username'], | |
275 | 'entries': tracks, | |
276 | } |