]>
Commit | Line | Data |
---|---|---|
28746fbd PH |
1 | # coding: utf-8 |
2 | from __future__ import unicode_literals | |
3 | ||
04b32c8f | 4 | import hashlib |
520e7533 | 5 | import re |
28746fbd PH |
6 | |
7 | from .common import InfoExtractor | |
bd8f48c7 YCH |
8 | from ..compat import ( |
9 | compat_parse_qs, | |
10 | compat_urlparse, | |
11 | ) | |
28746fbd | 12 | from ..utils import ( |
bd8f48c7 | 13 | ExtractorError, |
6461f2b7 YCH |
14 | int_or_none, |
15 | float_or_none, | |
bd8f48c7 YCH |
16 | parse_iso8601, |
17 | smuggle_url, | |
4bc15a68 | 18 | str_or_none, |
bd8f48c7 | 19 | strip_jsonp, |
04b32c8f | 20 | unified_timestamp, |
bd8f48c7 | 21 | unsmuggle_url, |
1f85029d | 22 | urlencode_postdata, |
28746fbd PH |
23 | ) |
24 | ||
25 | ||
26 | class BiliBiliIE(InfoExtractor): | |
b4eb08bb S |
27 | _VALID_URL = r'''(?x) |
28 | https?:// | |
29 | (?:(?:www|bangumi)\.)? | |
30 | bilibili\.(?:tv|com)/ | |
31 | (?: | |
32 | (?: | |
33 | video/[aA][vV]| | |
34 | anime/(?P<anime_id>\d+)/play\# | |
35 | )(?P<id_bv>\d+)| | |
36 | video/[bB][vV](?P<id>[^/?#&]+) | |
37 | ) | |
38 | ''' | |
28746fbd | 39 | |
bd8f48c7 | 40 | _TESTS = [{ |
28746fbd | 41 | 'url': 'http://www.bilibili.tv/video/av1074402/', |
3526c304 | 42 | 'md5': '5f7d29e1a2872f3df0cf76b1f87d3788', |
28746fbd | 43 | 'info_dict': { |
04b32c8f | 44 | 'id': '1074402', |
3526c304 | 45 | 'ext': 'flv', |
28746fbd | 46 | 'title': '【金坷垃】金泡沫', |
6461f2b7 | 47 | 'description': 'md5:ce18c2a2d2193f0df2917d270f2e5923', |
3526c304 S |
48 | 'duration': 308.067, |
49 | 'timestamp': 1398012678, | |
28746fbd | 50 | 'upload_date': '20140420', |
ec85ded8 | 51 | 'thumbnail': r're:^https?://.+\.jpg', |
d90e4030 | 52 | 'uploader': '菊子桑', |
6461f2b7 | 53 | 'uploader_id': '156160', |
28746fbd | 54 | }, |
bd8f48c7 YCH |
55 | }, { |
56 | # Tested in BiliBiliBangumiIE | |
57 | 'url': 'http://bangumi.bilibili.com/anime/1869/play#40062', | |
58 | 'only_matching': True, | |
59 | }, { | |
60 | 'url': 'http://bangumi.bilibili.com/anime/5802/play#100643', | |
61 | 'md5': '3f721ad1e75030cc06faf73587cfec57', | |
62 | 'info_dict': { | |
63 | 'id': '100643', | |
64 | 'ext': 'mp4', | |
65 | 'title': 'CHAOS;CHILD', | |
66 | 'description': '如果你是神明,并且能够让妄想成为现实。那你会进行怎么样的妄想?是淫靡的世界?独裁社会?毁灭性的制裁?还是……2015年,涩谷。从6年前发生的大灾害“涩谷地震”之后复兴了的这个街区里新设立的私立高中...', | |
67 | }, | |
68 | 'skip': 'Geo-restricted to China', | |
ca270371 YCH |
69 | }, { |
70 | # Title with double quotes | |
71 | 'url': 'http://www.bilibili.com/video/av8903802/', | |
72 | 'info_dict': { | |
73 | 'id': '8903802', | |
ca270371 YCH |
74 | 'title': '阿滴英文|英文歌分享#6 "Closer', |
75 | 'description': '滴妹今天唱Closer給你聽! 有史以来,被推最多次也是最久的歌曲,其实歌词跟我原本想像差蛮多的,不过还是好听! 微博@阿滴英文', | |
ca270371 | 76 | }, |
3526c304 S |
77 | 'playlist': [{ |
78 | 'info_dict': { | |
79 | 'id': '8903802_part1', | |
80 | 'ext': 'flv', | |
81 | 'title': '阿滴英文|英文歌分享#6 "Closer', | |
82 | 'description': 'md5:3b1b9e25b78da4ef87e9b548b88ee76a', | |
83 | 'uploader': '阿滴英文', | |
84 | 'uploader_id': '65880958', | |
85 | 'timestamp': 1488382634, | |
86 | 'upload_date': '20170301', | |
87 | }, | |
88 | 'params': { | |
89 | 'skip_download': True, # Test metadata only | |
90 | }, | |
91 | }, { | |
92 | 'info_dict': { | |
93 | 'id': '8903802_part2', | |
94 | 'ext': 'flv', | |
95 | 'title': '阿滴英文|英文歌分享#6 "Closer', | |
96 | 'description': 'md5:3b1b9e25b78da4ef87e9b548b88ee76a', | |
97 | 'uploader': '阿滴英文', | |
98 | 'uploader_id': '65880958', | |
99 | 'timestamp': 1488382634, | |
100 | 'upload_date': '20170301', | |
101 | }, | |
102 | 'params': { | |
103 | 'skip_download': True, # Test metadata only | |
104 | }, | |
105 | }] | |
b4eb08bb S |
106 | }, { |
107 | # new BV video id format | |
108 | 'url': 'https://www.bilibili.com/video/BV1JE411F741', | |
109 | 'only_matching': True, | |
bd8f48c7 | 110 | }] |
28746fbd | 111 | |
c9a0ea6e S |
112 | _APP_KEY = 'iVGUTjsxvpLeuDCf' |
113 | _BILIBILI_KEY = 'aHRmhWMLkdeMuILqORnYZocwMBpMEOdt' | |
6461f2b7 | 114 | |
bd8f48c7 YCH |
115 | def _report_error(self, result): |
116 | if 'message' in result: | |
117 | raise ExtractorError('%s said: %s' % (self.IE_NAME, result['message']), expected=True) | |
118 | elif 'code' in result: | |
119 | raise ExtractorError('%s returns error %d' % (self.IE_NAME, result['code']), expected=True) | |
120 | else: | |
121 | raise ExtractorError('Can\'t extract Bangumi episode ID') | |
122 | ||
520e7533 | 123 | def _real_extract(self, url): |
bd8f48c7 YCH |
124 | url, smuggled_data = unsmuggle_url(url, {}) |
125 | ||
126 | mobj = re.match(self._VALID_URL, url) | |
b4eb08bb | 127 | video_id = mobj.group('id') or mobj.group('id_bv') |
bd8f48c7 | 128 | anime_id = mobj.group('anime_id') |
6461f2b7 YCH |
129 | webpage = self._download_webpage(url, video_id) |
130 | ||
bd8f48c7 | 131 | if 'anime/' not in url: |
3526c304 | 132 | cid = self._search_regex( |
61cb6683 | 133 | r'\bcid(?:["\']:|=)(\d+)', webpage, 'cid', |
3526c304 S |
134 | default=None |
135 | ) or compat_parse_qs(self._search_regex( | |
95a1322b S |
136 | [r'EmbedPlayer\([^)]+,\s*"([^"]+)"\)', |
137 | r'EmbedPlayer\([^)]+,\s*\\"([^"]+)\\"\)', | |
138 | r'<iframe[^>]+src="https://secure\.bilibili\.com/secure,([^"]+)"'], | |
7be15d40 P |
139 | webpage, 'player parameters'))['cid'][0] |
140 | else: | |
bd8f48c7 | 141 | if 'no_bangumi_tip' not in smuggled_data: |
cefecac1 | 142 | self.to_screen('Downloading episode %s. To download all videos in anime %s, re-run youtube-dlc with %s' % ( |
bd8f48c7 YCH |
143 | video_id, anime_id, compat_urlparse.urljoin(url, '//bangumi.bilibili.com/anime/%s' % anime_id))) |
144 | headers = { | |
145 | 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', | |
3a513f29 | 146 | 'Referer': url |
bd8f48c7 YCH |
147 | } |
148 | headers.update(self.geo_verification_headers()) | |
149 | ||
1f85029d YCH |
150 | js = self._download_json( |
151 | 'http://bangumi.bilibili.com/web_api/get_source', video_id, | |
152 | data=urlencode_postdata({'episode_id': video_id}), | |
bd8f48c7 YCH |
153 | headers=headers) |
154 | if 'result' not in js: | |
155 | self._report_error(js) | |
7be15d40 | 156 | cid = js['result']['cid'] |
04b32c8f | 157 | |
3a513f29 LS |
158 | headers = { |
159 | 'Referer': url | |
160 | } | |
161 | headers.update(self.geo_verification_headers()) | |
162 | ||
d90e4030 | 163 | entries = [] |
c4a21bc9 | 164 | |
3526c304 S |
165 | RENDITIONS = ('qn=80&quality=80&type=', 'quality=2&type=mp4') |
166 | for num, rendition in enumerate(RENDITIONS, start=1): | |
167 | payload = 'appkey=%s&cid=%s&otype=json&%s' % (self._APP_KEY, cid, rendition) | |
168 | sign = hashlib.md5((payload + self._BILIBILI_KEY).encode('utf-8')).hexdigest() | |
169 | ||
170 | video_info = self._download_json( | |
d1239608 | 171 | 'http://interface.bilibili.com/v2/playurl?%s&sign=%s' % (payload, sign), |
3526c304 S |
172 | video_id, note='Downloading video info page', |
173 | headers=headers, fatal=num == len(RENDITIONS)) | |
174 | ||
175 | if not video_info: | |
176 | continue | |
177 | ||
178 | if 'durl' not in video_info: | |
179 | if num < len(RENDITIONS): | |
180 | continue | |
181 | self._report_error(video_info) | |
182 | ||
183 | for idx, durl in enumerate(video_info['durl']): | |
184 | formats = [{ | |
185 | 'url': durl['url'], | |
186 | 'filesize': int_or_none(durl['size']), | |
187 | }] | |
188 | for backup_url in durl.get('backup_url', []): | |
189 | formats.append({ | |
190 | 'url': backup_url, | |
191 | # backup URLs have lower priorities | |
192 | 'preference': -2 if 'hd.mp4' in backup_url else -3, | |
193 | }) | |
194 | ||
195 | for a_format in formats: | |
196 | a_format.setdefault('http_headers', {}).update({ | |
197 | 'Referer': url, | |
198 | }) | |
199 | ||
200 | self._sort_formats(formats) | |
201 | ||
202 | entries.append({ | |
203 | 'id': '%s_part%s' % (video_id, idx), | |
204 | 'duration': float_or_none(durl.get('length'), 1000), | |
205 | 'formats': formats, | |
6461f2b7 | 206 | }) |
3526c304 | 207 | break |
6461f2b7 | 208 | |
3526c304 S |
209 | title = self._html_search_regex( |
210 | ('<h1[^>]+\btitle=(["\'])(?P<title>(?:(?!\1).)+)\1', | |
211 | '(?s)<h1[^>]*>(?P<title>.+?)</h1>'), webpage, 'title', | |
212 | group='title') | |
6461f2b7 | 213 | description = self._html_search_meta('description', webpage) |
04b32c8f | 214 | timestamp = unified_timestamp(self._html_search_regex( |
3526c304 S |
215 | r'<time[^>]+datetime="([^"]+)"', webpage, 'upload time', |
216 | default=None) or self._html_search_meta( | |
217 | 'uploadDate', webpage, 'timestamp', default=None)) | |
1f85029d | 218 | thumbnail = self._html_search_meta(['og:image', 'thumbnailUrl'], webpage) |
6461f2b7 YCH |
219 | |
220 | # TODO 'view_count' requires deobfuscating Javascript | |
d90e4030 | 221 | info = { |
04b32c8f | 222 | 'id': video_id, |
d90e4030 | 223 | 'title': title, |
6461f2b7 YCH |
224 | 'description': description, |
225 | 'timestamp': timestamp, | |
7be15d40 | 226 | 'thumbnail': thumbnail, |
04b32c8f | 227 | 'duration': float_or_none(video_info.get('timelength'), scale=1000), |
28746fbd | 228 | } |
d90e4030 | 229 | |
6461f2b7 | 230 | uploader_mobj = re.search( |
3526c304 | 231 | r'<a[^>]+href="(?:https?:)?//space\.bilibili\.com/(?P<id>\d+)"[^>]*>(?P<name>[^<]+)', |
6461f2b7 YCH |
232 | webpage) |
233 | if uploader_mobj: | |
234 | info.update({ | |
235 | 'uploader': uploader_mobj.group('name'), | |
236 | 'uploader_id': uploader_mobj.group('id'), | |
237 | }) | |
3526c304 S |
238 | if not info.get('uploader'): |
239 | info['uploader'] = self._html_search_meta( | |
240 | 'author', webpage, 'uploader', default=None) | |
6461f2b7 YCH |
241 | |
242 | for entry in entries: | |
243 | entry.update(info) | |
244 | ||
d90e4030 | 245 | if len(entries) == 1: |
d90e4030 | 246 | return entries[0] |
247 | else: | |
ad73083f YCH |
248 | for idx, entry in enumerate(entries): |
249 | entry['id'] = '%s_part%d' % (video_id, (idx + 1)) | |
250 | ||
6461f2b7 | 251 | return { |
d90e4030 | 252 | '_type': 'multi_video', |
520e7533 | 253 | 'id': video_id, |
6461f2b7 YCH |
254 | 'title': title, |
255 | 'description': description, | |
d90e4030 | 256 | 'entries': entries, |
6461f2b7 | 257 | } |
bd8f48c7 YCH |
258 | |
259 | ||
260 | class BiliBiliBangumiIE(InfoExtractor): | |
261 | _VALID_URL = r'https?://bangumi\.bilibili\.com/anime/(?P<id>\d+)' | |
262 | ||
263 | IE_NAME = 'bangumi.bilibili.com' | |
264 | IE_DESC = 'BiliBili番剧' | |
265 | ||
266 | _TESTS = [{ | |
267 | 'url': 'http://bangumi.bilibili.com/anime/1869', | |
268 | 'info_dict': { | |
269 | 'id': '1869', | |
270 | 'title': '混沌武士', | |
271 | 'description': 'md5:6a9622b911565794c11f25f81d6a97d2', | |
272 | }, | |
273 | 'playlist_count': 26, | |
274 | }, { | |
275 | 'url': 'http://bangumi.bilibili.com/anime/1869', | |
276 | 'info_dict': { | |
277 | 'id': '1869', | |
278 | 'title': '混沌武士', | |
279 | 'description': 'md5:6a9622b911565794c11f25f81d6a97d2', | |
280 | }, | |
281 | 'playlist': [{ | |
282 | 'md5': '91da8621454dd58316851c27c68b0c13', | |
283 | 'info_dict': { | |
284 | 'id': '40062', | |
285 | 'ext': 'mp4', | |
286 | 'title': '混沌武士', | |
287 | 'description': '故事发生在日本的江户时代。风是一个小酒馆的打工女。一日,酒馆里来了一群恶霸,虽然他们的举动令风十分不满,但是毕竟风只是一届女流,无法对他们采取什么行动,只能在心里嘟哝。这时,酒家里又进来了个“不良份子...', | |
288 | 'timestamp': 1414538739, | |
289 | 'upload_date': '20141028', | |
290 | 'episode': '疾风怒涛 Tempestuous Temperaments', | |
291 | 'episode_number': 1, | |
292 | }, | |
293 | }], | |
294 | 'params': { | |
295 | 'playlist_items': '1', | |
296 | }, | |
297 | }] | |
298 | ||
299 | @classmethod | |
300 | def suitable(cls, url): | |
301 | return False if BiliBiliIE.suitable(url) else super(BiliBiliBangumiIE, cls).suitable(url) | |
302 | ||
303 | def _real_extract(self, url): | |
304 | bangumi_id = self._match_id(url) | |
305 | ||
306 | # Sometimes this API returns a JSONP response | |
307 | season_info = self._download_json( | |
308 | 'http://bangumi.bilibili.com/jsonp/seasoninfo/%s.ver' % bangumi_id, | |
309 | bangumi_id, transform_source=strip_jsonp)['result'] | |
310 | ||
311 | entries = [{ | |
312 | '_type': 'url_transparent', | |
313 | 'url': smuggle_url(episode['webplay_url'], {'no_bangumi_tip': 1}), | |
314 | 'ie_key': BiliBiliIE.ie_key(), | |
315 | 'timestamp': parse_iso8601(episode.get('update_time'), delimiter=' '), | |
316 | 'episode': episode.get('index_title'), | |
317 | 'episode_number': int_or_none(episode.get('index')), | |
318 | } for episode in season_info['episodes']] | |
319 | ||
320 | entries = sorted(entries, key=lambda entry: entry.get('episode_number')) | |
321 | ||
322 | return self.playlist_result( | |
323 | entries, bangumi_id, | |
324 | season_info.get('bangumi_title'), season_info.get('evaluate')) | |
4bc15a68 RA |
325 | |
326 | ||
327 | class BilibiliAudioBaseIE(InfoExtractor): | |
328 | def _call_api(self, path, sid, query=None): | |
329 | if not query: | |
330 | query = {'sid': sid} | |
331 | return self._download_json( | |
332 | 'https://www.bilibili.com/audio/music-service-c/web/' + path, | |
333 | sid, query=query)['data'] | |
334 | ||
335 | ||
336 | class BilibiliAudioIE(BilibiliAudioBaseIE): | |
337 | _VALID_URL = r'https?://(?:www\.)?bilibili\.com/audio/au(?P<id>\d+)' | |
338 | _TEST = { | |
339 | 'url': 'https://www.bilibili.com/audio/au1003142', | |
340 | 'md5': 'fec4987014ec94ef9e666d4d158ad03b', | |
341 | 'info_dict': { | |
342 | 'id': '1003142', | |
343 | 'ext': 'm4a', | |
344 | 'title': '【tsukimi】YELLOW / 神山羊', | |
345 | 'artist': 'tsukimi', | |
346 | 'comment_count': int, | |
347 | 'description': 'YELLOW的mp3版!', | |
348 | 'duration': 183, | |
349 | 'subtitles': { | |
350 | 'origin': [{ | |
351 | 'ext': 'lrc', | |
352 | }], | |
353 | }, | |
354 | 'thumbnail': r're:^https?://.+\.jpg', | |
355 | 'timestamp': 1564836614, | |
356 | 'upload_date': '20190803', | |
357 | 'uploader': 'tsukimi-つきみぐー', | |
358 | 'view_count': int, | |
359 | }, | |
360 | } | |
361 | ||
362 | def _real_extract(self, url): | |
363 | au_id = self._match_id(url) | |
364 | ||
365 | play_data = self._call_api('url', au_id) | |
366 | formats = [{ | |
367 | 'url': play_data['cdns'][0], | |
368 | 'filesize': int_or_none(play_data.get('size')), | |
369 | }] | |
370 | ||
371 | song = self._call_api('song/info', au_id) | |
372 | title = song['title'] | |
373 | statistic = song.get('statistic') or {} | |
374 | ||
375 | subtitles = None | |
376 | lyric = song.get('lyric') | |
377 | if lyric: | |
378 | subtitles = { | |
379 | 'origin': [{ | |
380 | 'url': lyric, | |
381 | }] | |
382 | } | |
383 | ||
384 | return { | |
385 | 'id': au_id, | |
386 | 'title': title, | |
387 | 'formats': formats, | |
388 | 'artist': song.get('author'), | |
389 | 'comment_count': int_or_none(statistic.get('comment')), | |
390 | 'description': song.get('intro'), | |
391 | 'duration': int_or_none(song.get('duration')), | |
392 | 'subtitles': subtitles, | |
393 | 'thumbnail': song.get('cover'), | |
394 | 'timestamp': int_or_none(song.get('passtime')), | |
395 | 'uploader': song.get('uname'), | |
396 | 'view_count': int_or_none(statistic.get('play')), | |
397 | } | |
398 | ||
399 | ||
400 | class BilibiliAudioAlbumIE(BilibiliAudioBaseIE): | |
401 | _VALID_URL = r'https?://(?:www\.)?bilibili\.com/audio/am(?P<id>\d+)' | |
402 | _TEST = { | |
403 | 'url': 'https://www.bilibili.com/audio/am10624', | |
404 | 'info_dict': { | |
405 | 'id': '10624', | |
406 | 'title': '每日新曲推荐(每日11:00更新)', | |
407 | 'description': '每天11:00更新,为你推送最新音乐', | |
408 | }, | |
409 | 'playlist_count': 19, | |
410 | } | |
411 | ||
412 | def _real_extract(self, url): | |
413 | am_id = self._match_id(url) | |
414 | ||
415 | songs = self._call_api( | |
416 | 'song/of-menu', am_id, {'sid': am_id, 'pn': 1, 'ps': 100})['data'] | |
417 | ||
418 | entries = [] | |
419 | for song in songs: | |
420 | sid = str_or_none(song.get('id')) | |
421 | if not sid: | |
422 | continue | |
423 | entries.append(self.url_result( | |
424 | 'https://www.bilibili.com/audio/au' + sid, | |
425 | BilibiliAudioIE.ie_key(), sid)) | |
426 | ||
427 | if entries: | |
428 | album_data = self._call_api('menu/info', am_id) or {} | |
429 | album_title = album_data.get('title') | |
430 | if album_title: | |
431 | for entry in entries: | |
432 | entry['album'] = album_title | |
433 | return self.playlist_result( | |
434 | entries, am_id, album_title, album_data.get('intro')) | |
435 | ||
436 | return self.playlist_result(entries, am_id) | |
63dce309 S |
437 | |
438 | ||
439 | class BiliBiliPlayerIE(InfoExtractor): | |
440 | _VALID_URL = r'https?://player\.bilibili\.com/player\.html\?.*?\baid=(?P<id>\d+)' | |
441 | _TEST = { | |
442 | 'url': 'http://player.bilibili.com/player.html?aid=92494333&cid=157926707&page=1', | |
443 | 'only_matching': True, | |
444 | } | |
445 | ||
446 | def _real_extract(self, url): | |
447 | video_id = self._match_id(url) | |
448 | return self.url_result( | |
449 | 'http://www.bilibili.tv/video/av%s/' % video_id, | |
450 | ie=BiliBiliIE.ie_key(), video_id=video_id) |