[yt-dlp.git] / youtube_dl / extractor / kuwo.py

# coding: utf-8
from __future__ import unicode_literals

import re
import itertools

from .common import InfoExtractor
from ..utils import (
    get_element_by_id,
    clean_html,
    ExtractorError,
)


class KuwoIE(InfoExtractor):
    IE_NAME = 'kuwo:song'
    _VALID_URL = r'http://www\.kuwo\.cn/yinyue/(?P<id>[0-9]+?)/'
    _TESTS = [{
        'url': 'http://www.kuwo.cn/yinyue/635632/',
        'info_dict': {
            'id': '635632',
            'ext': 'ape',
            'title': '爱我别走',
            'creator': '张震岳',
            'upload_date': '20080122',
            'description': 'md5:ed13f58e3c3bf3f7fd9fbc4e5a7aa75c'
        },
    }, {
        'url': 'http://www.kuwo.cn/yinyue/6446136/',
        'info_dict': {
            'id': '6446136',
            'ext': 'mp3',
            'title': '心',
            'creator': 'IU',
            'upload_date': '20150518',
        },
        'params': {
            'format': 'mp3-320'
        },
    }]
    _FORMATS = [
        {'format': 'ape', 'ext': 'ape', 'preference': 100},
        {'format': 'mp3-320', 'ext': 'mp3', 'br': '320kmp3', 'abr': 320, 'preference': 80},
        {'format': 'mp3-192', 'ext': 'mp3', 'br': '192kmp3', 'abr': 192, 'preference': 70},
        {'format': 'mp3-128', 'ext': 'mp3', 'br': '128kmp3', 'abr': 128, 'preference': 60},
        {'format': 'wma', 'ext': 'wma', 'preference': 20},
        {'format': 'aac', 'ext': 'aac', 'abr': 48, 'preference': 10}
    ]

    def _get_formats(self, song_id):
        formats = []
        for file_format in self._FORMATS:
            song_url = self._download_webpage(
                "http://antiserver.kuwo.cn/anti.s?format=%s&br=%s&rid=MUSIC_%s&type=convert_url&response=url" %
                (file_format['ext'], file_format.get('br', ''), song_id),
                song_id, note="Download %s url info" % file_format["format"],
            )
            if song_url.startswith('http://') or song_url.startswith('https://'):
                formats.append({
                    'url': song_url,
                    'format_id': file_format['format'],
                    'format': file_format['format'],
                    'preference': file_format['preference'],
                    'abr': file_format.get('abr'),
                })
        self._sort_formats(formats)
        return formats

    def _real_extract(self, url):
        song_id = self._match_id(url)
        webpage = self._download_webpage(
            url, song_id, note='Download song detail info',
            errnote='Unable to get song detail info')

        song_name = self._html_search_regex(
            r'<h1 title="(.+?)">', webpage, 'song name')
        singer_name = self._html_search_regex(
            r'<div class="s_img">.+?title="(.+?)".+?</div>', webpage, 'singer name',
            flags=re.DOTALL, default=None)
        lrc_content = clean_html(get_element_by_id("lrcContent", webpage))
        if lrc_content == '暂无':     # indicates no lyrics
            lrc_content = None

        formats = self._get_formats(song_id)

        album_id = self._html_search_regex(
            r'<p class="album" title=".+?">.+?<a href="http://www\.kuwo\.cn/album/([0-9]+)/" ',
            webpage, 'album id', default=None, fatal=False)

        publish_time = None
        if album_id is not None:
            album_info_page = self._download_webpage(
                "http://www.kuwo.cn/album/%s/" % album_id, song_id,
                note='Download album detail info',
                errnote='Unable to get album detail info')

            publish_time = self._html_search_regex(
                r'发行时间：(\d{4}-\d{2}-\d{2})', album_info_page,
                'publish time', default=None)
            if publish_time:
                publish_time = publish_time.replace('-', '')

        return {
            'id': song_id,
            'title': song_name,
            'creator': singer_name,
            'upload_date': publish_time,
            'description': lrc_content,
            'formats': formats,
        }


class KuwoAlbumIE(InfoExtractor):
    IE_NAME = 'kuwo:album'
    _VALID_URL = r'http://www\.kuwo\.cn/album/(?P<id>[0-9]+?)/'
    _TEST = {
        'url': 'http://www.kuwo.cn/album/502294/',
        'info_dict': {
            'id': '502294',
            'title': 'M',
            'description': 'md5:6a7235a84cc6400ec3b38a7bdaf1d60c',
        },
        'playlist_count': 2,
    }

    def _real_extract(self, url):
        album_id = self._match_id(url)

        webpage = self._download_webpage(
            url, album_id, note='Download album info',
            errnote='Unable to get album info')

        album_name = self._html_search_regex(
            r'<div class="comm".+?<h1 title="(.+?)">.+?</h1>', webpage,
            'album name', flags=re.DOTALL)
        album_intro = clean_html(
            re.sub(r'^.+简介：', '', get_element_by_id("intro", webpage).strip()))

        entries = [
            self.url_result("http://www.kuwo.cn/yinyue/%s/" % song_id, 'Kuwo', song_id)
            for song_id in re.findall(
                r'<p class="listen"><a href="http://www\.kuwo\.cn/yinyue/([0-9]+)/" target="_blank" title="试听.*?"></a></p>',
                webpage)
        ]
        return self.playlist_result(entries, album_id, album_name, album_intro)


class KuwoChartIE(InfoExtractor):
    IE_NAME = 'kuwo:chart'
    _VALID_URL = r'http://yinyue\.kuwo\.cn/billboard_(?P<id>.+?).htm'
    _TEST = {
        'url': 'http://yinyue.kuwo.cn/billboard_香港中文龙虎榜.htm',
        'info_dict': {
            'id': '香港中文龙虎榜',
            'title': '香港中文龙虎榜',
            'description': 're:[0-9]{4}第[0-9]{2}期',
        },
        'playlist_mincount': 10,
    }

    def _real_extract(self, url):
        chart_id = self._match_id(url)
        webpage = self._download_webpage(
            url, chart_id, note='Download chart info',
            errnote='Unable to get chart info')

        chart_name = self._html_search_regex(
            r'<h1 class="unDis">(.+?)</h1>', webpage, 'chart name')

        chart_desc = self._html_search_regex(
            r'<p class="tabDef">([0-9]{4}第[0-9]{2}期)</p>', webpage, 'chart desc')

        entries = [
            self.url_result("http://www.kuwo.cn/yinyue/%s/" % song_id, 'Kuwo', song_id)
            for song_id in re.findall(
                r'<a href="http://www\.kuwo\.cn/yinyue/([0-9]+)/" .+?>.+?</a>', webpage)
        ]
        return self.playlist_result(entries, chart_id, chart_name, chart_desc)


class KuwoSingerIE(InfoExtractor):
    IE_NAME = 'kuwo:singer'
    _VALID_URL = r'http://www\.kuwo\.cn/mingxing/(?P<id>[^/]+)'
    _TESTS = [{
        'url': 'http://www.kuwo.cn/mingxing/bruno+mars/',
        'info_dict': {
            'id': 'bruno+mars',
            'title': 'Bruno Mars',
        },
        'playlist_count': 10,
    }, {
        'url': 'http://www.kuwo.cn/mingxing/Ali/music.htm',
        'info_dict': {
            'id': 'Ali',
            'title': 'Ali',
        },
        'playlist_mincount': 95,
    }]

    def _real_extract(self, url):
        singer_id = self._match_id(url)
        webpage = self._download_webpage(
            url, singer_id, note='Download singer info',
            errnote='Unable to get singer info')

        singer_name = self._html_search_regex(
            r'<div class="title clearfix">[\n\s\t]*?<h1>(.+?)<span', webpage, 'singer name'
        )

        entries = []
        first_page_only = False if re.match(r'.+/music(?:_[0-9]+)?\.htm', url) else True
        for page_num in itertools.count(1):
            webpage = self._download_webpage(
                'http://www.kuwo.cn/mingxing/%s/music_%d.htm' % (singer_id, page_num),
                singer_id, note='Download song list page #%d' % page_num,
                errnote='Unable to get song list page #%d' % page_num)

            entries.extend([
                self.url_result("http://www.kuwo.cn/yinyue/%s/" % song_id, 'Kuwo', song_id)
                for song_id in re.findall(
                    r'<p class="m_name"><a href="http://www\.kuwo\.cn/yinyue/([0-9]+)/',
                    webpage)
            ][:10 if first_page_only else None])

            if first_page_only or not re.search(r'<a href="[^"]+">下一页</a>', webpage):
                break

        return self.playlist_result(entries, singer_id, singer_name)


class KuwoCategoryIE(InfoExtractor):
    IE_NAME = 'kuwo:category'
    _VALID_URL = r'http://yinyue\.kuwo\.cn/yy/cinfo_(?P<id>[0-9]+?).htm'
    _TEST = {
        'url': 'http://yinyue.kuwo.cn/yy/cinfo_86375.htm',
        'info_dict': {
            'id': '86375',
            'title': '八十年代精选',
            'description': '这些都是属于八十年代的回忆！',
        },
        'playlist_count': 30,
    }

    def _real_extract(self, url):
        category_id = self._match_id(url)
        webpage = self._download_webpage(
            url, category_id, note='Download category info',
            errnote='Unable to get category info')

        category_name = self._html_search_regex(
            r'<h1 title="([^<>]+?)">[^<>]+?</h1>', webpage, 'category name')
        
        category_desc = re.sub(
            r'^.+简介：', '', get_element_by_id("intro", webpage).strip())
        
        jsonm = self._parse_json(self._html_search_regex(
            r'var jsonm = (\{.+?\});', webpage, 'category songs'), category_id)

        entries = [
            self.url_result(
                "http://www.kuwo.cn/yinyue/%s/" % song['musicrid'],
                'Kuwo', song['musicrid'])
            for song in jsonm['musiclist']
        ]
        return self.playlist_result(entries, category_id, category_name, category_desc)


class KuwoMvIE(KuwoIE):
    IE_NAME = 'kuwo:mv'
    _VALID_URL = r'http://www\.kuwo\.cn/mv/(?P<id>[0-9]+?)/'
    _TESTS = [{
        'url': 'http://www.kuwo.cn/mv/6480076/',
        'info_dict': {
            'id': '6480076',
            'ext': 'mkv',
            'title': '我们家MV',
            'creator': '2PM',
        },
    }]
    _FORMATS = KuwoIE._FORMATS + [
        {'format': 'mkv', 'ext': 'mkv', 'preference': 250},
        {'format': 'mp4', 'ext': 'mp4', 'preference': 200},
    ]

    def _real_extract(self, url):
        song_id = self._match_id(url)
        webpage = self._download_webpage(
            url, song_id, note='Download mv detail info: %s' % song_id,
            errnote='Unable to get mv detail info: %s' % song_id)

        mobj = re.search(
            r'<h1 title="(?P<song>.+?)">[^<>]+<span .+?title="(?P<singer>.+?)".+?>[^<>]+</span></h1>',
            webpage)
        if mobj:
            song_name = mobj.group('song')
            singer_name = mobj.group('singer')
        else:
            raise ExtractorError("Unable to find song or singer names")

        formats = self._get_formats(song_id)

        return {
            'id': song_id,
            'title': song_name,
            'creator': singer_name,
            'formats': formats,
        }
Commit	Line	Data
8f73e89c	1	# coding: utf-8
	2	from __future__ import unicode_literals
	3
	4	import re
	5	import itertools
	6
	7	from .common import InfoExtractor
	8	from ..utils import (
	9	get_element_by_id,
	10	clean_html,
	11	ExtractorError,
	12	)
	13
	14
	15	class KuwoIE(InfoExtractor):
	16	IE_NAME = 'kuwo:song'
	17	_VALID_URL = r'http://www\.kuwo\.cn/yinyue/(?P<id>[0-9]+?)/'
	18	_TESTS = [{
	19	'url': 'http://www.kuwo.cn/yinyue/635632/',
	20	'info_dict': {
	21	'id': '635632',
	22	'ext': 'ape',
	23	'title': '爱我别走',
	24	'creator': '张震岳',
	25	'upload_date': '20080122',
	26	'description': 'md5:ed13f58e3c3bf3f7fd9fbc4e5a7aa75c'
	27	},
	28	}, {
	29	'url': 'http://www.kuwo.cn/yinyue/6446136/',
	30	'info_dict': {
	31	'id': '6446136',
	32	'ext': 'mp3',
	33	'title': '心',
	34	'creator': 'IU',
	35	'upload_date': '20150518',
	36	},
	37	'params': {
	38	'format': 'mp3-320'
	39	},
	40	}]
	41	_FORMATS = [
	42	{'format': 'ape', 'ext': 'ape', 'preference': 100},
	43	{'format': 'mp3-320', 'ext': 'mp3', 'br': '320kmp3', 'abr': 320, 'preference': 80},
	44	{'format': 'mp3-192', 'ext': 'mp3', 'br': '192kmp3', 'abr': 192, 'preference': 70},
	45	{'format': 'mp3-128', 'ext': 'mp3', 'br': '128kmp3', 'abr': 128, 'preference': 60},
	46	{'format': 'wma', 'ext': 'wma', 'preference': 20},
	47	{'format': 'aac', 'ext': 'aac', 'abr': 48, 'preference': 10}
	48	]
	49
	50	def _get_formats(self, song_id):
	51	formats = []
	52	for file_format in self._FORMATS:
	53	song_url = self._download_webpage(
	54	"http://antiserver.kuwo.cn/anti.s?format=%s&br=%s&rid=MUSIC_%s&type=convert_url&response=url" %
	55	(file_format['ext'], file_format.get('br', ''), song_id),
	56	song_id, note="Download %s url info" % file_format["format"],
	57	)
	58	if song_url.startswith('http://') or song_url.startswith('https://'):
	59	formats.append({
	60	'url': song_url,
	61	'format_id': file_format['format'],
	62	'format': file_format['format'],
	63	'preference': file_format['preference'],
	64	'abr': file_format.get('abr'),
65	})
66	self._sort_formats(formats)
67	return formats
68
69	def _real_extract(self, url):
70	song_id = self._match_id(url)
71	webpage = self._download_webpage(
72	url, song_id, note='Download song detail info',
73	errnote='Unable to get song detail info')
74
75	song_name = self._html_search_regex(
76	r'<h1 title="(.+?)">', webpage, 'song name')
77	singer_name = self._html_search_regex(
78	r'<div class="s_img">.+?title="(.+?)".+?</div>', webpage, 'singer name',
79	flags=re.DOTALL, default=None)
80	lrc_content = clean_html(get_element_by_id("lrcContent", webpage))
81	if lrc_content == '暂无': # indicates no lyrics
82	lrc_content = None
83
84	formats = self._get_formats(song_id)
85
86	album_id = self._html_search_regex(
87	r'<p class="album" title=".+?">.+?<a href="http://www\.kuwo\.cn/album/([0-9]+)/" ',
88	webpage, 'album id', default=None, fatal=False)
89
90	publish_time = None
91	if album_id is not None:
92	album_info_page = self._download_webpage(
93	"http://www.kuwo.cn/album/%s/" % album_id, song_id,
94	note='Download album detail info',
95	errnote='Unable to get album detail info')
96
97	publish_time = self._html_search_regex(
98	r'发行时间：(\d{4}-\d{2}-\d{2})', album_info_page,
99	'publish time', default=None)
100	if publish_time:
101	publish_time = publish_time.replace('-', '')
102
103	return {
104	'id': song_id,
105	'title': song_name,
106	'creator': singer_name,
107	'upload_date': publish_time,
108	'description': lrc_content,
109	'formats': formats,
110	}
111
112
113	class KuwoAlbumIE(InfoExtractor):
114	IE_NAME = 'kuwo:album'
115	_VALID_URL = r'http://www\.kuwo\.cn/album/(?P<id>[0-9]+?)/'
116	_TEST = {
117	'url': 'http://www.kuwo.cn/album/502294/',
118	'info_dict': {
119	'id': '502294',
120	'title': 'M',
121	'description': 'md5:6a7235a84cc6400ec3b38a7bdaf1d60c',
122	},
123	'playlist_count': 2,
124	}
125
126	def _real_extract(self, url):
127	album_id = self._match_id(url)
128
129	webpage = self._download_webpage(
130	url, album_id, note='Download album info',
131	errnote='Unable to get album info')
132
133	album_name = self._html_search_regex(
134	r'<div class="comm".+?<h1 title="(.+?)">.+?</h1>', webpage,
135	'album name', flags=re.DOTALL)
136	album_intro = clean_html(
137	re.sub(r'^.+简介：', '', get_element_by_id("intro", webpage).strip()))
138
139	entries = [
140	self.url_result("http://www.kuwo.cn/yinyue/%s/" % song_id, 'Kuwo', song_id)
141	for song_id in re.findall(
142	r'<p class="listen"><a href="http://www\.kuwo\.cn/yinyue/([0-9]+)/" target="_blank" title="试听.*?"></a></p>',
143	webpage)
144	]
145	return self.playlist_result(entries, album_id, album_name, album_intro)
146
147
148	class KuwoChartIE(InfoExtractor):
149	IE_NAME = 'kuwo:chart'
150	_VALID_URL = r'http://yinyue\.kuwo\.cn/billboard_(?P<id>.+?).htm'
151	_TEST = {
152	'url': 'http://yinyue.kuwo.cn/billboard_香港中文龙虎榜.htm',
153	'info_dict': {
154	'id': '香港中文龙虎榜',
155	'title': '香港中文龙虎榜',
156	'description': 're:[0-9]{4}第[0-9]{2}期',
157	},
158	'playlist_mincount': 10,
159	}
160
161	def _real_extract(self, url):
162	chart_id = self._match_id(url)
163	webpage = self._download_webpage(
164	url, chart_id, note='Download chart info',
165	errnote='Unable to get chart info')
166
167	chart_name = self._html_search_regex(
168	r'<h1 class="unDis">(.+?)</h1>', webpage, 'chart name')
169
170	chart_desc = self._html_search_regex(
171	r'<p class="tabDef">([0-9]{4}第[0-9]{2}期)</p>', webpage, 'chart desc')
172
173	entries = [
174	self.url_result("http://www.kuwo.cn/yinyue/%s/" % song_id, 'Kuwo', song_id)
175	for song_id in re.findall(
176	r'<a href="http://www\.kuwo\.cn/yinyue/([0-9]+)/" .+?>.+?</a>', webpage)
177	]
178	return self.playlist_result(entries, chart_id, chart_name, chart_desc)
179
180
181	class KuwoSingerIE(InfoExtractor):
182	IE_NAME = 'kuwo:singer'
1633491b	183	_VALID_URL = r'http://www\.kuwo\.cn/mingxing/(?P<id>[^/]+)'
1633491b	184	_TESTS = [{
8f73e89c	185	'url': 'http://www.kuwo.cn/mingxing/bruno+mars/',
	186	'info_dict': {
	187	'id': 'bruno+mars',
	188	'title': 'Bruno Mars',
	189	},
	190	'playlist_count': 10,
1633491b	191	}, {
8f73e89c	192	'url': 'http://www.kuwo.cn/mingxing/Ali/music.htm',
	193	'info_dict': {
	194	'id': 'Ali',
1633491b	195	'title': 'Ali',
8f73e89c	196	},
8f73e89c	197	'playlist_mincount': 95,
1633491b	198	}]
8f73e89c	199
	200	def _real_extract(self, url):
	201	singer_id = self._match_id(url)
1633491b	202	webpage = self._download_webpage(
	203	url, singer_id, note='Download singer info',
	204	errnote='Unable to get singer info')
	205
	206	singer_name = self._html_search_regex(
	207	r'<div class="title clearfix">[\n\s\t]*?<h1>(.+?)<span', webpage, 'singer name'
	208	)
8f73e89c	209
8f73e89c	210	entries = []
1633491b	211	first_page_only = False if re.match(r'.+/music(?:_[0-9]+)?\.htm', url) else True
8f73e89c	212	for page_num in itertools.count(1):
	213	webpage = self._download_webpage(
	214	'http://www.kuwo.cn/mingxing/%s/music_%d.htm' % (singer_id, page_num),
	215	singer_id, note='Download song list page #%d' % page_num,
	216	errnote='Unable to get song list page #%d' % page_num)
	217
8f73e89c	218	entries.extend([
	219	self.url_result("http://www.kuwo.cn/yinyue/%s/" % song_id, 'Kuwo', song_id)
	220	for song_id in re.findall(
	221	r'<p class="m_name"><a href="http://www\.kuwo\.cn/yinyue/([0-9]+)/',
	222	webpage)
1633491b	223	][:10 if first_page_only else None])
	224
	225	if first_page_only or not re.search(r'<a href="[^"]+">下一页</a>', webpage):
8f73e89c	226	break
8f73e89c	227
1633491b	228	return self.playlist_result(entries, singer_id, singer_name)
8f73e89c	229
	230
	231	class KuwoCategoryIE(InfoExtractor):
	232	IE_NAME = 'kuwo:category'
	233	_VALID_URL = r'http://yinyue\.kuwo\.cn/yy/cinfo_(?P<id>[0-9]+?).htm'
	234	_TEST = {
	235	'url': 'http://yinyue.kuwo.cn/yy/cinfo_86375.htm',
	236	'info_dict': {
	237	'id': '86375',
	238	'title': '八十年代精选',
	239	'description': '这些都是属于八十年代的回忆！',
	240	},
	241	'playlist_count': 30,
	242	}
	243
	244	def _real_extract(self, url):
	245	category_id = self._match_id(url)
	246	webpage = self._download_webpage(
	247	url, category_id, note='Download category info',
	248	errnote='Unable to get category info')
	249
	250	category_name = self._html_search_regex(
	251	r'<h1 title="([^<>]+?)">[^<>]+?</h1>', webpage, 'category name')
	252
	253	category_desc = re.sub(
	254	r'^.+简介：', '', get_element_by_id("intro", webpage).strip())
	255
	256	jsonm = self._parse_json(self._html_search_regex(
	257	r'var jsonm = (\{.+?\});', webpage, 'category songs'), category_id)
	258
	259	entries = [
	260	self.url_result(
	261	"http://www.kuwo.cn/yinyue/%s/" % song['musicrid'],
	262	'Kuwo', song['musicrid'])
	263	for song in jsonm['musiclist']
	264	]
	265	return self.playlist_result(entries, category_id, category_name, category_desc)
	266
	267
	268	class KuwoMvIE(KuwoIE):
	269	IE_NAME = 'kuwo:mv'
	270	_VALID_URL = r'http://www\.kuwo\.cn/mv/(?P<id>[0-9]+?)/'
	271	_TESTS = [{
	272	'url': 'http://www.kuwo.cn/mv/6480076/',
	273	'info_dict': {
	274	'id': '6480076',
	275	'ext': 'mkv',
	276	'title': '我们家MV',
	277	'creator': '2PM',
	278	},
	279	}]
	280	_FORMATS = KuwoIE._FORMATS + [
	281	{'format': 'mkv', 'ext': 'mkv', 'preference': 250},
	282	{'format': 'mp4', 'ext': 'mp4', 'preference': 200},
	283	]
	284
	285	def _real_extract(self, url):
	286	song_id = self._match_id(url)
	287	webpage = self._download_webpage(
	288	url, song_id, note='Download mv detail info: %s' % song_id,
	289	errnote='Unable to get mv detail info: %s' % song_id)
	290
	291	mobj = re.search(
	292	r'<h1 title="(?P<song>.+?)">[^<>]+<span .+?title="(?P<singer>.+?)".+?>[^<>]+</span></h1>',
293	webpage)
294	if mobj:
295	song_name = mobj.group('song')
296	singer_name = mobj.group('singer')
297	else:
298	raise ExtractorError("Unable to find song or singer names")
299
300	formats = self._get_formats(song_id)
301
302	return {
303	'id': song_id,
304	'title': song_name,
305	'creator': singer_name,
306	'formats': formats,
307	}