yt_dlp/extractor/neteasemusic.py

   1 import itertools
   2 import re
   3 from base64 import b64encode
   4 from datetime import datetime
   5 from hashlib import md5
   6
   7 from .common import InfoExtractor
   8 from ..compat import compat_str, compat_urllib_parse_urlencode
   9 from ..utils import float_or_none, sanitized_Request
  10
  11
  12 class NetEaseMusicBaseIE(InfoExtractor):
  13     _FORMATS = ['bMusic', 'mMusic', 'hMusic']
  14     _NETEASE_SALT = '3go8&$8*3*3h0k(2)2'
  15     _API_BASE = 'http://music.163.com/api/'
  16
  17     @classmethod
  18     def _encrypt(cls, dfsid):
  19         salt_bytes = bytearray(cls._NETEASE_SALT.encode('utf-8'))
  20         string_bytes = bytearray(compat_str(dfsid).encode('ascii'))
  21         salt_len = len(salt_bytes)
  22         for i in range(len(string_bytes)):
  23             string_bytes[i] = string_bytes[i] ^ salt_bytes[i % salt_len]
  24         m = md5()
  25         m.update(bytes(string_bytes))
  26         result = b64encode(m.digest()).decode('ascii')
  27         return result.replace('/', '_').replace('+', '-')
  28
  29     def extract_formats(self, info):
  30         formats = []
  31         for song_format in self._FORMATS:
  32             details = info.get(song_format)
  33             if not details:
  34                 continue
  35             song_file_path = '/%s/%s.%s' % (
  36                 self._encrypt(details['dfsId']), details['dfsId'], details['extension'])
  37
  38             # 203.130.59.9, 124.40.233.182, 115.231.74.139, etc is a reverse proxy-like feature
  39             # from NetEase's CDN provider that can be used if m5.music.126.net does not
  40             # work, especially for users outside of Mainland China
  41             # via: https://github.com/JixunMoe/unblock-163/issues/3#issuecomment-163115880
  42             for host in ('http://m5.music.126.net', 'http://115.231.74.139/m1.music.126.net',
  43                          'http://124.40.233.182/m1.music.126.net', 'http://203.130.59.9/m1.music.126.net'):
  44                 song_url = host + song_file_path
  45                 if self._is_valid_url(song_url, info['id'], 'song'):
  46                     formats.append({
  47                         'url': song_url,
  48                         'ext': details.get('extension'),
  49                         'abr': float_or_none(details.get('bitrate'), scale=1000),
  50                         'format_id': song_format,
  51                         'filesize': details.get('size'),
  52                         'asr': details.get('sr')
  53                     })
  54                     break
  55         return formats
  56
  57     @classmethod
  58     def convert_milliseconds(cls, ms):
  59         return int(round(ms / 1000.0))
  60
  61     def query_api(self, endpoint, video_id, note):
  62         req = sanitized_Request('%s%s' % (self._API_BASE, endpoint))
  63         req.add_header('Referer', self._API_BASE)
  64         return self._download_json(req, video_id, note)
  65
  66
  67 class NetEaseMusicIE(NetEaseMusicBaseIE):
  68     IE_NAME = 'netease:song'
  69     IE_DESC = '网易云音乐'
  70     _VALID_URL = r'https?://music\.163\.com/(#/)?song\?id=(?P<id>[0-9]+)'
  71     _TESTS = [{
  72         'url': 'http://music.163.com/#/song?id=32102397',
  73         'md5': 'f2e97280e6345c74ba9d5677dd5dcb45',
  74         'info_dict': {
  75             'id': '32102397',
  76             'ext': 'mp3',
  77             'title': 'Bad Blood (feat. Kendrick Lamar)',
  78             'creator': 'Taylor Swift / Kendrick Lamar',
  79             'upload_date': '20150517',
  80             'timestamp': 1431878400,
  81             'description': 'md5:a10a54589c2860300d02e1de821eb2ef',
  82         },
  83         'skip': 'Blocked outside Mainland China',
  84     }, {
  85         'note': 'No lyrics translation.',
  86         'url': 'http://music.163.com/#/song?id=29822014',
  87         'info_dict': {
  88             'id': '29822014',
  89             'ext': 'mp3',
  90             'title': '听见下雨的声音',
  91             'creator': '周杰伦',
  92             'upload_date': '20141225',
  93             'timestamp': 1419523200,
  94             'description': 'md5:a4d8d89f44656af206b7b2555c0bce6c',
  95         },
  96         'skip': 'Blocked outside Mainland China',
  97     }, {
  98         'note': 'No lyrics.',
  99         'url': 'http://music.163.com/song?id=17241424',
 100         'info_dict': {
 101             'id': '17241424',
 102             'ext': 'mp3',
 103             'title': 'Opus 28',
 104             'creator': 'Dustin O\'Halloran',
 105             'upload_date': '20080211',
 106             'timestamp': 1202745600,
 107         },
 108         'skip': 'Blocked outside Mainland China',
 109     }, {
 110         'note': 'Has translated name.',
 111         'url': 'http://music.163.com/#/song?id=22735043',
 112         'info_dict': {
 113             'id': '22735043',
 114             'ext': 'mp3',
 115             'title': '소원을 말해봐 (Genie)',
 116             'creator': '少女时代',
 117             'description': 'md5:79d99cc560e4ca97e0c4d86800ee4184',
 118             'upload_date': '20100127',
 119             'timestamp': 1264608000,
 120             'alt_title': '说出愿望吧(Genie)',
 121         },
 122         'skip': 'Blocked outside Mainland China',
 123     }]
 124
 125     def _process_lyrics(self, lyrics_info):
 126         original = lyrics_info.get('lrc', {}).get('lyric')
 127         translated = lyrics_info.get('tlyric', {}).get('lyric')
 128
 129         if not translated:
 130             return original
 131
 132         lyrics_expr = r'(\[[0-9]{2}:[0-9]{2}\.[0-9]{2,}\])([^\n]+)'
 133         original_ts_texts = re.findall(lyrics_expr, original)
 134         translation_ts_dict = dict(
 135             (time_stamp, text) for time_stamp, text in re.findall(lyrics_expr, translated)
 136         )
 137         lyrics = '\n'.join([
 138             '%s%s / %s' % (time_stamp, text, translation_ts_dict.get(time_stamp, ''))
 139             for time_stamp, text in original_ts_texts
 140         ])
 141         return lyrics
 142
 143     def _real_extract(self, url):
 144         song_id = self._match_id(url)
 145
 146         params = {
 147             'id': song_id,
 148             'ids': '[%s]' % song_id
 149         }
 150         info = self.query_api(
 151             'song/detail?' + compat_urllib_parse_urlencode(params),
 152             song_id, 'Downloading song info')['songs'][0]
 153
 154         formats = self.extract_formats(info)
 155         self._sort_formats(formats)
 156
 157         lyrics_info = self.query_api(
 158             'song/lyric?id=%s&lv=-1&tv=-1' % song_id,
 159             song_id, 'Downloading lyrics data')
 160         lyrics = self._process_lyrics(lyrics_info)
 161
 162         alt_title = None
 163         if info.get('transNames'):
 164             alt_title = '/'.join(info.get('transNames'))
 165
 166         return {
 167             'id': song_id,
 168             'title': info['name'],
 169             'alt_title': alt_title,
 170             'creator': ' / '.join([artist['name'] for artist in info.get('artists', [])]),
 171             'timestamp': self.convert_milliseconds(info.get('album', {}).get('publishTime')),
 172             'thumbnail': info.get('album', {}).get('picUrl'),
 173             'duration': self.convert_milliseconds(info.get('duration', 0)),
 174             'description': lyrics,
 175             'formats': formats,
 176         }
 177
 178
 179 class NetEaseMusicAlbumIE(NetEaseMusicBaseIE):
 180     IE_NAME = 'netease:album'
 181     IE_DESC = '网易云音乐 - 专辑'
 182     _VALID_URL = r'https?://music\.163\.com/(#/)?album\?id=(?P<id>[0-9]+)'
 183     _TEST = {
 184         'url': 'http://music.163.com/#/album?id=220780',
 185         'info_dict': {
 186             'id': '220780',
 187             'title': 'B\'day',
 188         },
 189         'playlist_count': 23,
 190         'skip': 'Blocked outside Mainland China',
 191     }
 192
 193     def _real_extract(self, url):
 194         album_id = self._match_id(url)
 195
 196         info = self.query_api(
 197             'album/%s?id=%s' % (album_id, album_id),
 198             album_id, 'Downloading album data')['album']
 199
 200         name = info['name']
 201         desc = info.get('description')
 202         entries = [
 203             self.url_result('http://music.163.com/#/song?id=%s' % song['id'],
 204                             'NetEaseMusic', song['id'])
 205             for song in info['songs']
 206         ]
 207         return self.playlist_result(entries, album_id, name, desc)
 208
 209
 210 class NetEaseMusicSingerIE(NetEaseMusicBaseIE):
 211     IE_NAME = 'netease:singer'
 212     IE_DESC = '网易云音乐 - 歌手'
 213     _VALID_URL = r'https?://music\.163\.com/(#/)?artist\?id=(?P<id>[0-9]+)'
 214     _TESTS = [{
 215         'note': 'Singer has aliases.',
 216         'url': 'http://music.163.com/#/artist?id=10559',
 217         'info_dict': {
 218             'id': '10559',
 219             'title': '张惠妹 - aMEI;阿密特',
 220         },
 221         'playlist_count': 50,
 222         'skip': 'Blocked outside Mainland China',
 223     }, {
 224         'note': 'Singer has translated name.',
 225         'url': 'http://music.163.com/#/artist?id=124098',
 226         'info_dict': {
 227             'id': '124098',
 228             'title': '李昇基 - 이승기',
 229         },
 230         'playlist_count': 50,
 231         'skip': 'Blocked outside Mainland China',
 232     }]
 233
 234     def _real_extract(self, url):
 235         singer_id = self._match_id(url)
 236
 237         info = self.query_api(
 238             'artist/%s?id=%s' % (singer_id, singer_id),
 239             singer_id, 'Downloading singer data')
 240
 241         name = info['artist']['name']
 242         if info['artist']['trans']:
 243             name = '%s - %s' % (name, info['artist']['trans'])
 244         if info['artist']['alias']:
 245             name = '%s - %s' % (name, ';'.join(info['artist']['alias']))
 246
 247         entries = [
 248             self.url_result('http://music.163.com/#/song?id=%s' % song['id'],
 249                             'NetEaseMusic', song['id'])
 250             for song in info['hotSongs']
 251         ]
 252         return self.playlist_result(entries, singer_id, name)
 253
 254
 255 class NetEaseMusicListIE(NetEaseMusicBaseIE):
 256     IE_NAME = 'netease:playlist'
 257     IE_DESC = '网易云音乐 - 歌单'
 258     _VALID_URL = r'https?://music\.163\.com/(#/)?(playlist|discover/toplist)\?id=(?P<id>[0-9]+)'
 259     _TESTS = [{
 260         'url': 'http://music.163.com/#/playlist?id=79177352',
 261         'info_dict': {
 262             'id': '79177352',
 263             'title': 'Billboard 2007 Top 100',
 264             'description': 'md5:12fd0819cab2965b9583ace0f8b7b022'
 265         },
 266         'playlist_count': 99,
 267         'skip': 'Blocked outside Mainland China',
 268     }, {
 269         'note': 'Toplist/Charts sample',
 270         'url': 'http://music.163.com/#/discover/toplist?id=3733003',
 271         'info_dict': {
 272             'id': '3733003',
 273             'title': 're:韩国Melon排行榜周榜 [0-9]{4}-[0-9]{2}-[0-9]{2}',
 274             'description': 'md5:73ec782a612711cadc7872d9c1e134fc',
 275         },
 276         'playlist_count': 50,
 277         'skip': 'Blocked outside Mainland China',
 278     }]
 279
 280     def _real_extract(self, url):
 281         list_id = self._match_id(url)
 282
 283         info = self.query_api(
 284             'playlist/detail?id=%s&lv=-1&tv=-1' % list_id,
 285             list_id, 'Downloading playlist data')['result']
 286
 287         name = info['name']
 288         desc = info.get('description')
 289
 290         if info.get('specialType') == 10:  # is a chart/toplist
 291             datestamp = datetime.fromtimestamp(
 292                 self.convert_milliseconds(info['updateTime'])).strftime('%Y-%m-%d')
 293             name = '%s %s' % (name, datestamp)
 294
 295         entries = [
 296             self.url_result('http://music.163.com/#/song?id=%s' % song['id'],
 297                             'NetEaseMusic', song['id'])
 298             for song in info['tracks']
 299         ]
 300         return self.playlist_result(entries, list_id, name, desc)
 301
 302
 303 class NetEaseMusicMvIE(NetEaseMusicBaseIE):
 304     IE_NAME = 'netease:mv'
 305     IE_DESC = '网易云音乐 - MV'
 306     _VALID_URL = r'https?://music\.163\.com/(#/)?mv\?id=(?P<id>[0-9]+)'
 307     _TEST = {
 308         'url': 'http://music.163.com/#/mv?id=415350',
 309         'info_dict': {
 310             'id': '415350',
 311             'ext': 'mp4',
 312             'title': '이럴거면 그러지말지',
 313             'description': '白雅言自作曲唱甜蜜爱情',
 314             'creator': '白雅言',
 315             'upload_date': '20150520',
 316         },
 317         'skip': 'Blocked outside Mainland China',
 318     }
 319
 320     def _real_extract(self, url):
 321         mv_id = self._match_id(url)
 322
 323         info = self.query_api(
 324             'mv/detail?id=%s&type=mp4' % mv_id,
 325             mv_id, 'Downloading mv info')['data']
 326
 327         formats = [
 328             {'url': mv_url, 'ext': 'mp4', 'format_id': '%sp' % brs, 'height': int(brs)}
 329             for brs, mv_url in info['brs'].items()
 330         ]
 331         self._sort_formats(formats)
 332
 333         return {
 334             'id': mv_id,
 335             'title': info['name'],
 336             'description': info.get('desc') or info.get('briefDesc'),
 337             'creator': info['artistName'],
 338             'upload_date': info['publishTime'].replace('-', ''),
 339             'formats': formats,
 340             'thumbnail': info.get('cover'),
 341             'duration': self.convert_milliseconds(info.get('duration', 0)),
 342         }
 343
 344
 345 class NetEaseMusicProgramIE(NetEaseMusicBaseIE):
 346     IE_NAME = 'netease:program'
 347     IE_DESC = '网易云音乐 - 电台节目'
 348     _VALID_URL = r'https?://music\.163\.com/(#/?)program\?id=(?P<id>[0-9]+)'
 349     _TESTS = [{
 350         'url': 'http://music.163.com/#/program?id=10109055',
 351         'info_dict': {
 352             'id': '10109055',
 353             'ext': 'mp3',
 354             'title': '不丹足球背后的故事',
 355             'description': '喜马拉雅人的足球梦 ...',
 356             'creator': '大话西藏',
 357             'timestamp': 1434179342,
 358             'upload_date': '20150613',
 359             'duration': 900,
 360         },
 361         'skip': 'Blocked outside Mainland China',
 362     }, {
 363         'note': 'This program has accompanying songs.',
 364         'url': 'http://music.163.com/#/program?id=10141022',
 365         'info_dict': {
 366             'id': '10141022',
 367             'title': '25岁，你是自在如风的少年<27°C>',
 368             'description': 'md5:8d594db46cc3e6509107ede70a4aaa3b',
 369         },
 370         'playlist_count': 4,
 371         'skip': 'Blocked outside Mainland China',
 372     }, {
 373         'note': 'This program has accompanying songs.',
 374         'url': 'http://music.163.com/#/program?id=10141022',
 375         'info_dict': {
 376             'id': '10141022',
 377             'ext': 'mp3',
 378             'title': '25岁，你是自在如风的少年<27°C>',
 379             'description': 'md5:8d594db46cc3e6509107ede70a4aaa3b',
 380             'timestamp': 1434450841,
 381             'upload_date': '20150616',
 382         },
 383         'params': {
 384             'noplaylist': True
 385         },
 386         'skip': 'Blocked outside Mainland China',
 387     }]
 388
 389     def _real_extract(self, url):
 390         program_id = self._match_id(url)
 391
 392         info = self.query_api(
 393             'dj/program/detail?id=%s' % program_id,
 394             program_id, 'Downloading program info')['program']
 395
 396         name = info['name']
 397         description = info['description']
 398
 399         if not self._yes_playlist(info['songs'] and program_id, info['mainSong']['id']):
 400             formats = self.extract_formats(info['mainSong'])
 401             self._sort_formats(formats)
 402
 403             return {
 404                 'id': info['mainSong']['id'],
 405                 'title': name,
 406                 'description': description,
 407                 'creator': info['dj']['brand'],
 408                 'timestamp': self.convert_milliseconds(info['createTime']),
 409                 'thumbnail': info['coverUrl'],
 410                 'duration': self.convert_milliseconds(info.get('duration', 0)),
 411                 'formats': formats,
 412             }
 413
 414         song_ids = [info['mainSong']['id']]
 415         song_ids.extend([song['id'] for song in info['songs']])
 416         entries = [
 417             self.url_result('http://music.163.com/#/song?id=%s' % song_id,
 418                             'NetEaseMusic', song_id)
 419             for song_id in song_ids
 420         ]
 421         return self.playlist_result(entries, program_id, name, description)
 422
 423
 424 class NetEaseMusicDjRadioIE(NetEaseMusicBaseIE):
 425     IE_NAME = 'netease:djradio'
 426     IE_DESC = '网易云音乐 - 电台'
 427     _VALID_URL = r'https?://music\.163\.com/(#/)?djradio\?id=(?P<id>[0-9]+)'
 428     _TEST = {
 429         'url': 'http://music.163.com/#/djradio?id=42',
 430         'info_dict': {
 431             'id': '42',
 432             'title': '声音蔓延',
 433             'description': 'md5:766220985cbd16fdd552f64c578a6b15'
 434         },
 435         'playlist_mincount': 40,
 436         'skip': 'Blocked outside Mainland China',
 437     }
 438     _PAGE_SIZE = 1000
 439
 440     def _real_extract(self, url):
 441         dj_id = self._match_id(url)
 442
 443         name = None
 444         desc = None
 445         entries = []
 446         for offset in itertools.count(start=0, step=self._PAGE_SIZE):
 447             info = self.query_api(
 448                 'dj/program/byradio?asc=false&limit=%d&radioId=%s&offset=%d'
 449                 % (self._PAGE_SIZE, dj_id, offset),
 450                 dj_id, 'Downloading dj programs - %d' % offset)
 451
 452             entries.extend([
 453                 self.url_result(
 454                     'http://music.163.com/#/program?id=%s' % program['id'],
 455                     'NetEaseMusicProgram', program['id'])
 456                 for program in info['programs']
 457             ])
 458
 459             if name is None:
 460                 radio = info['programs'][0]['radio']
 461                 name = radio['name']
 462                 desc = radio['desc']
 463
 464             if not info['more']:
 465                 break
 466
 467         return self.playlist_result(entries, dj_id, name, desc)