]>
Commit | Line | Data |
---|---|---|
9eb4f404 S |
1 | # coding: utf-8 |
2 | from __future__ import unicode_literals | |
f58487b3 TV |
3 | |
4 | import re | |
cc1ac110 | 5 | import itertools |
f58487b3 | 6 | |
9eb4f404 | 7 | from .common import InfoExtractor |
36b226d4 | 8 | from ..utils import ( |
9 | urlencode_postdata, | |
10 | int_or_none, | |
11 | unified_strdate, | |
12 | ) | |
9eb4f404 S |
13 | |
14 | ||
15 | class VierIE(InfoExtractor): | |
16 | IE_NAME = 'vier' | |
e129fa08 | 17 | IE_DESC = 'vier.be and vijf.be' |
0a2e1b2e S |
18 | _VALID_URL = r'''(?x) |
19 | https?:// | |
20 | (?:www\.)?(?P<site>vier|vijf)\.be/ | |
21 | (?: | |
22 | (?: | |
23 | [^/]+/videos| | |
24 | video(?:/[^/]+)* | |
25 | )/ | |
26 | (?P<display_id>[^/]+)(?:/(?P<id>\d+))?| | |
27 | (?: | |
28 | video/v3/embed| | |
29 | embed/video/public | |
30 | )/(?P<embed_id>\d+) | |
31 | ) | |
32 | ''' | |
89fd0307 | 33 | _NETRC_MACHINE = 'vier' |
9eb4f404 S |
34 | _TESTS = [{ |
35 | 'url': 'http://www.vier.be/planb/videos/het-wordt-warm-de-moestuin/16129', | |
7073015a | 36 | 'md5': 'e4ae2054a6b040ef1e289e20d111b46e', |
9eb4f404 S |
37 | 'info_dict': { |
38 | 'id': '16129', | |
39 | 'display_id': 'het-wordt-warm-de-moestuin', | |
40 | 'ext': 'mp4', | |
41 | 'title': 'Het wordt warm in De Moestuin', | |
42 | 'description': 'De vele uren werk eisen hun tol. Wim droomt van assistentie...', | |
36b226d4 | 43 | 'upload_date': '20121025', |
650bd947 S |
44 | 'series': 'Plan B', |
45 | 'tags': ['De Moestuin', 'Moestuin', 'meisjes', 'Tomaat', 'Wim', 'Droom'], | |
9eb4f404 | 46 | }, |
a3ba8a7a LV |
47 | }, { |
48 | 'url': 'http://www.vijf.be/temptationisland/videos/zo-grappig-temptation-island-hosts-moeten-kiezen-tussen-onmogelijke-dilemmas/2561614', | |
49 | 'info_dict': { | |
50 | 'id': '2561614', | |
51 | 'display_id': 'zo-grappig-temptation-island-hosts-moeten-kiezen-tussen-onmogelijke-dilemmas', | |
52 | 'ext': 'mp4', | |
7073015a S |
53 | 'title': 'md5:84f45fe48b8c1fa296a7f6d208d080a7', |
54 | 'description': 'md5:0356d4981e58b8cbee19355cbd51a8fe', | |
36b226d4 | 55 | 'upload_date': '20170228', |
650bd947 S |
56 | 'series': 'Temptation Island', |
57 | 'tags': list, | |
a3ba8a7a LV |
58 | }, |
59 | 'params': { | |
a3ba8a7a LV |
60 | 'skip_download': True, |
61 | }, | |
89fd0307 | 62 | }, { |
63 | 'url': 'http://www.vier.be/janigaat/videos/jani-gaat-naar-tokio-aflevering-4/2674839', | |
64 | 'info_dict': { | |
65 | 'id': '2674839', | |
66 | 'display_id': 'jani-gaat-naar-tokio-aflevering-4', | |
67 | 'ext': 'mp4', | |
68 | 'title': 'Jani gaat naar Tokio - Aflevering 4', | |
36b226d4 | 69 | 'description': 'md5:aa8d611541db6ae9e863125704511f88', |
70 | 'upload_date': '20170501', | |
650bd947 | 71 | 'series': 'Jani gaat', |
36b226d4 | 72 | 'episode_number': 4, |
650bd947 | 73 | 'tags': ['Jani Gaat', 'Volledige Aflevering'], |
89fd0307 | 74 | }, |
75 | 'params': { | |
89fd0307 | 76 | 'skip_download': True, |
77 | }, | |
78 | 'skip': 'Requires account credentials', | |
79 | }, { | |
7073015a S |
80 | # Requires account credentials but bypassed extraction via v3/embed page |
81 | # without metadata | |
89fd0307 | 82 | 'url': 'http://www.vier.be/janigaat/videos/jani-gaat-naar-tokio-aflevering-4/2674839', |
83 | 'info_dict': { | |
84 | 'id': '2674839', | |
85 | 'display_id': 'jani-gaat-naar-tokio-aflevering-4', | |
86 | 'ext': 'mp4', | |
87 | 'title': 'jani-gaat-naar-tokio-aflevering-4', | |
88 | }, | |
89 | 'params': { | |
89fd0307 | 90 | 'skip_download': True, |
91 | }, | |
92 | 'expected_warnings': ['Log in to extract metadata'], | |
9eb4f404 | 93 | }, { |
7073015a S |
94 | # Without video id in URL |
95 | 'url': 'http://www.vier.be/planb/videos/dit-najaar-plan-b', | |
9eb4f404 S |
96 | 'only_matching': True, |
97 | }, { | |
98 | 'url': 'http://www.vier.be/video/v3/embed/16129', | |
99 | 'only_matching': True, | |
0a2e1b2e S |
100 | }, { |
101 | 'url': 'https://www.vijf.be/embed/video/public/4093', | |
102 | 'only_matching': True, | |
103 | }, { | |
104 | 'url': 'https://www.vier.be/video/blockbusters/in-juli-en-augustus-summer-classics', | |
105 | 'only_matching': True, | |
106 | }, { | |
107 | 'url': 'https://www.vier.be/video/achter-de-rug/2017/achter-de-rug-seizoen-1-aflevering-6', | |
108 | 'only_matching': True, | |
9eb4f404 S |
109 | }] |
110 | ||
89fd0307 | 111 | def _real_initialize(self): |
112 | self._logged_in = False | |
113 | ||
114 | def _login(self, site): | |
115 | username, password = self._get_login_info() | |
116 | if username is None or password is None: | |
117 | return | |
118 | ||
119 | login_page = self._download_webpage( | |
120 | 'http://www.%s.be/user/login' % site, | |
121 | None, note='Logging in', errnote='Unable to log in', | |
122 | data=urlencode_postdata({ | |
123 | 'form_id': 'user_login', | |
124 | 'name': username, | |
125 | 'pass': password, | |
126 | }), | |
127 | headers={'Content-Type': 'application/x-www-form-urlencoded'}) | |
128 | ||
129 | login_error = self._html_search_regex( | |
130 | r'(?s)<div class="messages error">\s*<div>\s*<h2.+?</h2>(.+?)<', | |
131 | login_page, 'login error', default=None) | |
132 | if login_error: | |
133 | self.report_warning('Unable to log in: %s' % login_error) | |
134 | else: | |
135 | self._logged_in = True | |
136 | ||
9eb4f404 S |
137 | def _real_extract(self, url): |
138 | mobj = re.match(self._VALID_URL, url) | |
139 | embed_id = mobj.group('embed_id') | |
140 | display_id = mobj.group('display_id') or embed_id | |
89fd0307 | 141 | video_id = mobj.group('id') or embed_id |
a3ba8a7a | 142 | site = mobj.group('site') |
9eb4f404 | 143 | |
89fd0307 | 144 | if not self._logged_in: |
145 | self._login(site) | |
146 | ||
9eb4f404 S |
147 | webpage = self._download_webpage(url, display_id) |
148 | ||
89fd0307 | 149 | if r'id="user-login"' in webpage: |
150 | self.report_warning( | |
151 | 'Log in to extract metadata', video_id=display_id) | |
152 | webpage = self._download_webpage( | |
153 | 'http://www.%s.be/video/v3/embed/%s' % (site, video_id), | |
154 | display_id) | |
155 | ||
9eb4f404 | 156 | video_id = self._search_regex( |
484c9d2d | 157 | [r'data-nid="(\d+)"', r'"nid"\s*:\s*"(\d+)"'], |
7073015a | 158 | webpage, 'video id', default=video_id or display_id) |
0a2e1b2e S |
159 | |
160 | playlist_url = self._search_regex( | |
161 | r'data-file=(["\'])(?P<url>(?:https?:)?//[^/]+/.+?\.m3u8.*?)\1', | |
162 | webpage, 'm3u8 url', default=None, group='url') | |
163 | ||
164 | if not playlist_url: | |
165 | application = self._search_regex( | |
166 | [r'data-application="([^"]+)"', r'"application"\s*:\s*"([^"]+)"'], | |
167 | webpage, 'application', default=site + '_vod') | |
168 | filename = self._search_regex( | |
169 | [r'data-filename="([^"]+)"', r'"filename"\s*:\s*"([^"]+)"'], | |
170 | webpage, 'filename') | |
171 | playlist_url = 'http://vod.streamcloud.be/%s/_definst_/mp4:%s.mp4/playlist.m3u8' % (application, filename) | |
172 | ||
650bd947 S |
173 | formats = self._extract_wowza_formats( |
174 | playlist_url, display_id, skip_protocols=['dash']) | |
19dbaeec | 175 | self._sort_formats(formats) |
9eb4f404 S |
176 | |
177 | title = self._og_search_title(webpage, default=display_id) | |
36b226d4 | 178 | description = self._html_search_regex( |
650bd947 S |
179 | r'(?s)<div\b[^>]+\bclass=(["\'])[^>]*?\bfield-type-text-with-summary\b[^>]*?\1[^>]*>.*?<p>(?P<value>.+?)</p>', |
180 | webpage, 'description', default=None, group='value') | |
181 | thumbnail = self._og_search_thumbnail(webpage, default=None) | |
36b226d4 | 182 | upload_date = unified_strdate(self._html_search_regex( |
650bd947 S |
183 | r'(?s)<div\b[^>]+\bclass=(["\'])[^>]*?\bfield-name-post-date\b[^>]*?\1[^>]*>.*?(?P<value>\d{2}/\d{2}/\d{4})', |
184 | webpage, 'upload date', default=None, group='value')) | |
185 | ||
186 | series = self._search_regex( | |
187 | r'data-program=(["\'])(?P<value>(?:(?!\1).)+)\1', webpage, | |
188 | 'series', default=None, group='value') | |
189 | episode_number = int_or_none(self._search_regex( | |
190 | r'(?i)aflevering (\d+)', title, 'episode number', default=None)) | |
191 | tags = re.findall(r'<a\b[^>]+\bhref=["\']/tags/[^>]+>([^<]+)<', webpage) | |
9eb4f404 S |
192 | |
193 | return { | |
194 | 'id': video_id, | |
195 | 'display_id': display_id, | |
196 | 'title': title, | |
197 | 'description': description, | |
198 | 'thumbnail': thumbnail, | |
650bd947 S |
199 | 'upload_date': upload_date, |
200 | 'series': series, | |
201 | 'episode_number': episode_number, | |
202 | 'tags': tags, | |
9eb4f404 | 203 | 'formats': formats, |
f58487b3 | 204 | } |
9eb4f404 S |
205 | |
206 | ||
207 | class VierVideosIE(InfoExtractor): | |
208 | IE_NAME = 'vier:videos' | |
a3ba8a7a | 209 | _VALID_URL = r'https?://(?:www\.)?(?P<site>vier|vijf)\.be/(?P<program>[^/]+)/videos(?:\?.*\bpage=(?P<page>\d+)|$)' |
9eb4f404 S |
210 | _TESTS = [{ |
211 | 'url': 'http://www.vier.be/demoestuin/videos', | |
212 | 'info_dict': { | |
213 | 'id': 'demoestuin', | |
214 | }, | |
215 | 'playlist_mincount': 153, | |
a3ba8a7a LV |
216 | }, { |
217 | 'url': 'http://www.vijf.be/temptationisland/videos', | |
218 | 'info_dict': { | |
219 | 'id': 'temptationisland', | |
220 | }, | |
221 | 'playlist_mincount': 159, | |
9eb4f404 S |
222 | }, { |
223 | 'url': 'http://www.vier.be/demoestuin/videos?page=6', | |
224 | 'info_dict': { | |
225 | 'id': 'demoestuin-page6', | |
226 | }, | |
227 | 'playlist_mincount': 20, | |
228 | }, { | |
229 | 'url': 'http://www.vier.be/demoestuin/videos?page=7', | |
230 | 'info_dict': { | |
231 | 'id': 'demoestuin-page7', | |
232 | }, | |
233 | 'playlist_mincount': 13, | |
234 | }] | |
235 | ||
236 | def _real_extract(self, url): | |
237 | mobj = re.match(self._VALID_URL, url) | |
238 | program = mobj.group('program') | |
a3ba8a7a | 239 | site = mobj.group('site') |
9eb4f404 | 240 | |
9eb4f404 S |
241 | page_id = mobj.group('page') |
242 | if page_id: | |
243 | page_id = int(page_id) | |
244 | start_page = page_id | |
9eb4f404 S |
245 | playlist_id = '%s-page%d' % (program, page_id) |
246 | else: | |
247 | start_page = 0 | |
9eb4f404 S |
248 | playlist_id = program |
249 | ||
250 | entries = [] | |
cc1ac110 | 251 | for current_page_id in itertools.count(start_page): |
9eb4f404 | 252 | current_page = self._download_webpage( |
a3ba8a7a | 253 | 'http://www.%s.be/%s/videos?page=%d' % (site, program, current_page_id), |
9eb4f404 | 254 | program, |
cc1ac110 | 255 | 'Downloading page %d' % (current_page_id + 1)) |
9eb4f404 | 256 | page_entries = [ |
a3ba8a7a | 257 | self.url_result('http://www.' + site + '.be' + video_url, 'Vier') |
9eb4f404 | 258 | for video_url in re.findall( |
a3ba8a7a | 259 | r'<h[23]><a href="(/[^/]+/videos/[^/]+(?:/\d+)?)">', current_page)] |
9eb4f404 | 260 | entries.extend(page_entries) |
cc1ac110 S |
261 | if page_id or '>Meer<' not in current_page: |
262 | break | |
9eb4f404 S |
263 | |
264 | return self.playlist_result(entries, playlist_id) |