]>
Commit | Line | Data |
---|---|---|
5f6a1245 | 1 | # coding: utf-8 |
22a6f150 PH |
2 | from __future__ import unicode_literals |
3 | ||
219b8130 | 4 | import re |
b27c856f | 5 | import json |
a3c736de | 6 | import itertools |
219b8130 PH |
7 | |
8 | from .common import InfoExtractor | |
953e32b2 | 9 | |
1cc79574 | 10 | from ..compat import ( |
953e32b2 | 11 | compat_str, |
1cc79574 PH |
12 | compat_urllib_request, |
13 | ) | |
14 | from ..utils import ( | |
15 | ExtractorError, | |
16 | int_or_none, | |
c3fef636 | 17 | orderedSet, |
f53c966a | 18 | str_to_int, |
4b10aadf | 19 | unescapeHTML, |
219b8130 PH |
20 | ) |
21 | ||
5f6a1245 | 22 | |
70922df8 JMF |
23 | class DailymotionBaseInfoExtractor(InfoExtractor): |
24 | @staticmethod | |
25 | def _build_request(url): | |
26 | """Build a request with the family filter disabled""" | |
27 | request = compat_urllib_request.Request(url) | |
2a0c2ca2 | 28 | request.add_header('Cookie', 'family_filter=off; ff=off') |
70922df8 | 29 | return request |
953e32b2 | 30 | |
5f6a1245 | 31 | |
a1f2a06b | 32 | class DailymotionIE(DailymotionBaseInfoExtractor): |
219b8130 PH |
33 | """Information Extractor for Dailymotion""" |
34 | ||
9ee859b6 | 35 | _VALID_URL = r'(?i)(?:https?://)?(?:(www|touch)\.)?dailymotion\.[a-z]{2,3}/(?:(embed|#)/)?video/(?P<id>[^/?_]+)' |
ce6815aa | 36 | IE_NAME = 'dailymotion' |
cdec0190 JMF |
37 | |
38 | _FORMATS = [ | |
ce6815aa PH |
39 | ('stream_h264_ld_url', 'ld'), |
40 | ('stream_h264_url', 'standard'), | |
41 | ('stream_h264_hq_url', 'hq'), | |
42 | ('stream_h264_hd_url', 'hd'), | |
43 | ('stream_h264_hd1080_url', 'hd180'), | |
cdec0190 JMF |
44 | ] |
45 | ||
c5428382 JMF |
46 | _TESTS = [ |
47 | { | |
23ba76bc JMF |
48 | 'url': 'https://www.dailymotion.com/video/x2iuewm_steam-machine-models-pricing-listed-on-steam-store-ign-news_videogames', |
49 | 'md5': '2137c41a8e78554bb09225b8eb322406', | |
ce6815aa | 50 | 'info_dict': { |
23ba76bc | 51 | 'id': 'x2iuewm', |
ce6815aa | 52 | 'ext': 'mp4', |
23ba76bc JMF |
53 | 'uploader': 'IGN', |
54 | 'title': 'Steam Machine Models, Pricing Listed on Steam Store - IGN News', | |
a909e6ad | 55 | 'upload_date': '20150306', |
50aa2bb6 | 56 | 'duration': 74, |
c5428382 JMF |
57 | } |
58 | }, | |
59 | # Vevo video | |
60 | { | |
ce6815aa PH |
61 | 'url': 'http://www.dailymotion.com/video/x149uew_katy-perry-roar-official_musi', |
62 | 'info_dict': { | |
63 | 'title': 'Roar (Official)', | |
64 | 'id': 'USUV71301934', | |
65 | 'ext': 'mp4', | |
66 | 'uploader': 'Katy Perry', | |
67 | 'upload_date': '20130905', | |
c5428382 | 68 | }, |
ce6815aa PH |
69 | 'params': { |
70 | 'skip_download': True, | |
c5428382 | 71 | }, |
ce6815aa | 72 | 'skip': 'VEVO is only available in some countries', |
c5428382 | 73 | }, |
9f1109a5 PH |
74 | # age-restricted video |
75 | { | |
ce6815aa PH |
76 | 'url': 'http://www.dailymotion.com/video/xyh2zz_leanna-decker-cyber-girl-of-the-year-desires-nude-playboy-plus_redband', |
77 | 'md5': '0d667a7b9cebecc3c89ee93099c4159d', | |
78 | 'info_dict': { | |
79 | 'id': 'xyh2zz', | |
80 | 'ext': 'mp4', | |
81 | 'title': 'Leanna Decker - Cyber Girl Of The Year Desires Nude [Playboy Plus]', | |
82 | 'uploader': 'HotWaves1012', | |
83 | 'age_limit': 18, | |
9f1109a5 | 84 | } |
9f1109a5 | 85 | } |
c5428382 | 86 | ] |
219b8130 PH |
87 | |
88 | def _real_extract(self, url): | |
b10609d9 | 89 | video_id = self._match_id(url) |
5477ca82 | 90 | url = 'https://www.dailymotion.com/video/%s' % video_id |
219b8130 PH |
91 | |
92 | # Retrieve video webpage to extract further information | |
70922df8 | 93 | request = self._build_request(url) |
219b8130 PH |
94 | webpage = self._download_webpage(request, video_id) |
95 | ||
96 | # Extract URL, uploader and title from webpage | |
97 | self.report_extraction(video_id) | |
219b8130 | 98 | |
c5428382 JMF |
99 | # It may just embed a vevo video: |
100 | m_vevo = re.search( | |
9e05d039 | 101 | r'<link rel="video_src" href="[^"]*?vevo.com[^"]*?video=(?P<id>[\w]*)', |
c5428382 JMF |
102 | webpage) |
103 | if m_vevo is not None: | |
104 | vevo_id = m_vevo.group('id') | |
22a6f150 PH |
105 | self.to_screen('Vevo video detected: %s' % vevo_id) |
106 | return self.url_result('vevo:%s' % vevo_id, ie='Vevo') | |
c5428382 | 107 | |
9f1109a5 | 108 | age_limit = self._rta_search(webpage) |
219b8130 PH |
109 | |
110 | video_upload_date = None | |
a909e6ad | 111 | mobj = re.search(r'<meta property="video:release_date" content="([0-9]{4})-([0-9]{2})-([0-9]{2}).+?"/>', webpage) |
219b8130 | 112 | if mobj is not None: |
a909e6ad | 113 | video_upload_date = mobj.group(1) + mobj.group(2) + mobj.group(3) |
219b8130 | 114 | |
5477ca82 | 115 | embed_url = 'https://www.dailymotion.com/embed/video/%s' % video_id |
2a0c2ca2 S |
116 | embed_request = self._build_request(embed_url) |
117 | embed_page = self._download_webpage( | |
118 | embed_request, video_id, 'Downloading embed page') | |
4ff7a0f1 | 119 | info = self._search_regex(r'var info = ({.*?}),$', embed_page, |
9e1a5b84 | 120 | 'video info', flags=re.MULTILINE) |
b27c856f | 121 | info = json.loads(info) |
3a1d48d6 JMF |
122 | if info.get('error') is not None: |
123 | msg = 'Couldn\'t get video, Dailymotion says: %s' % info['error']['title'] | |
124 | raise ExtractorError(msg, expected=True) | |
b27c856f | 125 | |
cdec0190 JMF |
126 | formats = [] |
127 | for (key, format_id) in self._FORMATS: | |
128 | video_url = info.get(key) | |
129 | if video_url is not None: | |
130 | m_size = re.search(r'H264-(\d+)x(\d+)', video_url) | |
131 | if m_size is not None: | |
553f6e46 | 132 | width, height = map(int_or_none, (m_size.group(1), m_size.group(2))) |
cdec0190 JMF |
133 | else: |
134 | width, height = None, None | |
135 | formats.append({ | |
136 | 'url': video_url, | |
137 | 'ext': 'mp4', | |
138 | 'format_id': format_id, | |
139 | 'width': width, | |
140 | 'height': height, | |
141 | }) | |
142 | if not formats: | |
22a6f150 | 143 | raise ExtractorError('Unable to extract video URL') |
b27c856f | 144 | |
953e32b2 | 145 | # subtitles |
1f343eaa | 146 | video_subtitles = self.extract_subtitles(video_id, webpage) |
953e32b2 | 147 | |
b10609d9 PH |
148 | view_count = str_to_int(self._search_regex( |
149 | r'video_views_count[^>]+>\s+([\d\.,]+)', | |
150 | webpage, 'view count', fatal=False)) | |
151 | ||
152 | title = self._og_search_title(webpage, default=None) | |
153 | if title is None: | |
154 | title = self._html_search_regex( | |
155 | r'(?s)<span\s+id="video_title"[^>]*>(.*?)</span>', webpage, | |
156 | 'title') | |
f53c966a | 157 | |
9f1109a5 | 158 | return { |
b10609d9 | 159 | 'id': video_id, |
cdec0190 | 160 | 'formats': formats, |
85342674 | 161 | 'uploader': info['owner.screenname'], |
b10609d9 PH |
162 | 'upload_date': video_upload_date, |
163 | 'title': title, | |
164 | 'subtitles': video_subtitles, | |
9f1109a5 PH |
165 | 'thumbnail': info['thumbnail_url'], |
166 | 'age_limit': age_limit, | |
f53c966a | 167 | 'view_count': view_count, |
50aa2bb6 | 168 | 'duration': info['duration'] |
9f1109a5 | 169 | } |
a3c736de | 170 | |
a1f2a06b | 171 | def _get_subtitles(self, video_id, webpage): |
f8e52269 | 172 | try: |
7fad1c63 JMF |
173 | sub_list = self._download_webpage( |
174 | 'https://api.dailymotion.com/video/%s/subtitles?fields=id,language,url' % video_id, | |
175 | video_id, note=False) | |
176 | except ExtractorError as err: | |
22a6f150 | 177 | self._downloader.report_warning('unable to download video subtitles: %s' % compat_str(err)) |
f8e52269 IM |
178 | return {} |
179 | info = json.loads(sub_list) | |
180 | if (info['total'] > 0): | |
a1f2a06b | 181 | sub_lang_list = dict((l['language'], [{'url': l['url'], 'ext': 'srt'}]) for l in info['list']) |
f8e52269 | 182 | return sub_lang_list |
22a6f150 | 183 | self._downloader.report_warning('video doesn\'t have subtitles') |
f8e52269 IM |
184 | return {} |
185 | ||
a3c736de | 186 | |
70922df8 | 187 | class DailymotionPlaylistIE(DailymotionBaseInfoExtractor): |
22a6f150 | 188 | IE_NAME = 'dailymotion:playlist' |
a3c736de | 189 | _VALID_URL = r'(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/playlist/(?P<id>.+?)/' |
1e0a235f | 190 | _MORE_PAGES_INDICATOR = r'(?s)<div class="pages[^"]*">.*?<a\s+class="[^"]*?icon-arrow_right[^"]*?"' |
39baacc4 | 191 | _PAGE_TEMPLATE = 'https://www.dailymotion.com/playlist/%s/%s' |
22a6f150 PH |
192 | _TESTS = [{ |
193 | 'url': 'http://www.dailymotion.com/playlist/xv4bw_nqtv_sport/1#video=xl8v3q', | |
194 | 'info_dict': { | |
195 | 'title': 'SPORT', | |
11e611a7 | 196 | 'id': 'xv4bw_nqtv_sport', |
22a6f150 PH |
197 | }, |
198 | 'playlist_mincount': 20, | |
199 | }] | |
a3c736de | 200 | |
39baacc4 | 201 | def _extract_entries(self, id): |
a3c736de | 202 | video_ids = [] |
a3c736de | 203 | for pagenum in itertools.count(1): |
70922df8 JMF |
204 | request = self._build_request(self._PAGE_TEMPLATE % (id, pagenum)) |
205 | webpage = self._download_webpage(request, | |
22a6f150 | 206 | id, 'Downloading page %s' % pagenum) |
a3c736de | 207 | |
4b10aadf | 208 | video_ids.extend(re.findall(r'data-xid="(.+?)"', webpage)) |
a3c736de | 209 | |
1e0a235f | 210 | if re.search(self._MORE_PAGES_INDICATOR, webpage) is None: |
a3c736de | 211 | break |
39baacc4 | 212 | return [self.url_result('http://www.dailymotion.com/video/%s' % video_id, 'Dailymotion') |
9e1a5b84 | 213 | for video_id in orderedSet(video_ids)] |
39baacc4 JMF |
214 | |
215 | def _real_extract(self, url): | |
216 | mobj = re.match(self._VALID_URL, url) | |
217 | playlist_id = mobj.group('id') | |
218 | webpage = self._download_webpage(url, playlist_id) | |
219 | ||
b0fb63ab PH |
220 | return { |
221 | '_type': 'playlist', | |
222 | 'id': playlist_id, | |
223 | 'title': self._og_search_title(webpage), | |
224 | 'entries': self._extract_entries(playlist_id), | |
225 | } | |
39baacc4 JMF |
226 | |
227 | ||
228 | class DailymotionUserIE(DailymotionPlaylistIE): | |
22a6f150 | 229 | IE_NAME = 'dailymotion:user' |
5406af92 | 230 | _VALID_URL = r'https?://(?:www\.)?dailymotion\.[a-z]{2,3}/(?:(?:old/)?user/)?(?P<user>[^/]+)$' |
39baacc4 | 231 | _PAGE_TEMPLATE = 'http://www.dailymotion.com/user/%s/%s' |
22a6f150 PH |
232 | _TESTS = [{ |
233 | 'url': 'https://www.dailymotion.com/user/nqtv', | |
234 | 'info_dict': { | |
235 | 'id': 'nqtv', | |
236 | 'title': 'Rémi Gaillard', | |
237 | }, | |
238 | 'playlist_mincount': 100, | |
239 | }] | |
39baacc4 JMF |
240 | |
241 | def _real_extract(self, url): | |
242 | mobj = re.match(self._VALID_URL, url) | |
243 | user = mobj.group('user') | |
7d65242d S |
244 | webpage = self._download_webpage( |
245 | 'https://www.dailymotion.com/user/%s' % user, user) | |
4b10aadf S |
246 | full_user = unescapeHTML(self._html_search_regex( |
247 | r'<a class="nav-image" title="([^"]+)" href="/%s">' % re.escape(user), | |
22a6f150 | 248 | webpage, 'user')) |
39baacc4 JMF |
249 | |
250 | return { | |
251 | '_type': 'playlist', | |
252 | 'id': user, | |
253 | 'title': full_user, | |
254 | 'entries': self._extract_entries(user), | |
255 | } | |
756f574e YCH |
256 | |
257 | ||
258 | class DailymotionCloudIE(DailymotionBaseInfoExtractor): | |
0bcdc276 S |
259 | _VALID_URL_PREFIX = r'http://api\.dmcloud\.net/(?:player/)?embed/' |
260 | _VALID_URL = r'%s[^/]+/(?P<id>[^/?]+)' % _VALID_URL_PREFIX | |
261 | _VALID_EMBED_URL = r'%s[^/]+/[^\'"]+' % _VALID_URL_PREFIX | |
756f574e | 262 | |
0bcdc276 | 263 | _TESTS = [{ |
756f574e YCH |
264 | # From http://www.francetvinfo.fr/economie/entreprises/les-entreprises-familiales-le-secret-de-la-reussite_933271.html |
265 | # Tested at FranceTvInfo_2 | |
266 | 'url': 'http://api.dmcloud.net/embed/4e7343f894a6f677b10006b4/556e03339473995ee145930c?auth=1464865870-0-jyhsm84b-ead4c701fb750cf9367bf4447167a3db&autoplay=1', | |
267 | 'only_matching': True, | |
0bcdc276 S |
268 | }, { |
269 | # http://www.francetvinfo.fr/societe/larguez-les-amarres-le-cobaturage-se-developpe_980101.html | |
270 | 'url': 'http://api.dmcloud.net/player/embed/4e7343f894a6f677b10006b4/559545469473996d31429f06?auth=1467430263-0-90tglw2l-a3a4b64ed41efe48d7fccad85b8b8fda&autoplay=1', | |
271 | 'only_matching': True, | |
272 | }] | |
756f574e YCH |
273 | |
274 | @classmethod | |
275 | def _extract_dmcloud_url(self, webpage): | |
0bcdc276 | 276 | mobj = re.search(r'<iframe[^>]+src=[\'"](%s)[\'"]' % self._VALID_EMBED_URL, webpage) |
756f574e YCH |
277 | if mobj: |
278 | return mobj.group(1) | |
279 | ||
0bcdc276 S |
280 | mobj = re.search( |
281 | r'<input[^>]+id=[\'"]dmcloudUrlEmissionSelect[\'"][^>]+value=[\'"](%s)[\'"]' % self._VALID_EMBED_URL, | |
282 | webpage) | |
756f574e YCH |
283 | if mobj: |
284 | return mobj.group(1) | |
285 | ||
286 | def _real_extract(self, url): | |
287 | video_id = self._match_id(url) | |
288 | ||
289 | request = self._build_request(url) | |
290 | webpage = self._download_webpage(request, video_id) | |
291 | ||
292 | title = self._html_search_regex(r'<title>([^>]+)</title>', webpage, 'title') | |
293 | ||
294 | video_info = self._parse_json(self._search_regex( | |
295 | r'var\s+info\s*=\s*([^;]+);', webpage, 'video info'), video_id) | |
296 | ||
297 | # TODO: parse ios_url, which is in fact a manifest | |
298 | video_url = video_info['mp4_url'] | |
299 | ||
300 | return { | |
301 | 'id': video_id, | |
302 | 'url': video_url, | |
303 | 'title': title, | |
304 | 'thumbnail': video_info.get('thumbnail_url'), | |
305 | } |