]>
Commit | Line | Data |
---|---|---|
f58487b3 | 1 | import re |
cc1ac110 | 2 | import itertools |
f58487b3 | 3 | |
9eb4f404 | 4 | from .common import InfoExtractor |
36b226d4 | 5 | from ..utils import ( |
6 | urlencode_postdata, | |
7 | int_or_none, | |
8 | unified_strdate, | |
9 | ) | |
9eb4f404 S |
10 | |
11 | ||
12 | class VierIE(InfoExtractor): | |
13 | IE_NAME = 'vier' | |
e129fa08 | 14 | IE_DESC = 'vier.be and vijf.be' |
0a2e1b2e S |
15 | _VALID_URL = r'''(?x) |
16 | https?:// | |
17 | (?:www\.)?(?P<site>vier|vijf)\.be/ | |
18 | (?: | |
19 | (?: | |
20 | [^/]+/videos| | |
21 | video(?:/[^/]+)* | |
22 | )/ | |
23 | (?P<display_id>[^/]+)(?:/(?P<id>\d+))?| | |
24 | (?: | |
25 | video/v3/embed| | |
26 | embed/video/public | |
27 | )/(?P<embed_id>\d+) | |
28 | ) | |
29 | ''' | |
89fd0307 | 30 | _NETRC_MACHINE = 'vier' |
9eb4f404 S |
31 | _TESTS = [{ |
32 | 'url': 'http://www.vier.be/planb/videos/het-wordt-warm-de-moestuin/16129', | |
7073015a | 33 | 'md5': 'e4ae2054a6b040ef1e289e20d111b46e', |
9eb4f404 S |
34 | 'info_dict': { |
35 | 'id': '16129', | |
36 | 'display_id': 'het-wordt-warm-de-moestuin', | |
37 | 'ext': 'mp4', | |
38 | 'title': 'Het wordt warm in De Moestuin', | |
39 | 'description': 'De vele uren werk eisen hun tol. Wim droomt van assistentie...', | |
36b226d4 | 40 | 'upload_date': '20121025', |
650bd947 S |
41 | 'series': 'Plan B', |
42 | 'tags': ['De Moestuin', 'Moestuin', 'meisjes', 'Tomaat', 'Wim', 'Droom'], | |
9eb4f404 | 43 | }, |
a3ba8a7a LV |
44 | }, { |
45 | 'url': 'http://www.vijf.be/temptationisland/videos/zo-grappig-temptation-island-hosts-moeten-kiezen-tussen-onmogelijke-dilemmas/2561614', | |
46 | 'info_dict': { | |
47 | 'id': '2561614', | |
48 | 'display_id': 'zo-grappig-temptation-island-hosts-moeten-kiezen-tussen-onmogelijke-dilemmas', | |
49 | 'ext': 'mp4', | |
7073015a S |
50 | 'title': 'md5:84f45fe48b8c1fa296a7f6d208d080a7', |
51 | 'description': 'md5:0356d4981e58b8cbee19355cbd51a8fe', | |
36b226d4 | 52 | 'upload_date': '20170228', |
650bd947 S |
53 | 'series': 'Temptation Island', |
54 | 'tags': list, | |
a3ba8a7a LV |
55 | }, |
56 | 'params': { | |
a3ba8a7a LV |
57 | 'skip_download': True, |
58 | }, | |
89fd0307 | 59 | }, { |
60 | 'url': 'http://www.vier.be/janigaat/videos/jani-gaat-naar-tokio-aflevering-4/2674839', | |
61 | 'info_dict': { | |
62 | 'id': '2674839', | |
63 | 'display_id': 'jani-gaat-naar-tokio-aflevering-4', | |
64 | 'ext': 'mp4', | |
65 | 'title': 'Jani gaat naar Tokio - Aflevering 4', | |
36b226d4 | 66 | 'description': 'md5:aa8d611541db6ae9e863125704511f88', |
67 | 'upload_date': '20170501', | |
650bd947 | 68 | 'series': 'Jani gaat', |
36b226d4 | 69 | 'episode_number': 4, |
650bd947 | 70 | 'tags': ['Jani Gaat', 'Volledige Aflevering'], |
89fd0307 | 71 | }, |
72 | 'params': { | |
89fd0307 | 73 | 'skip_download': True, |
74 | }, | |
75 | 'skip': 'Requires account credentials', | |
76 | }, { | |
7073015a S |
77 | # Requires account credentials but bypassed extraction via v3/embed page |
78 | # without metadata | |
89fd0307 | 79 | 'url': 'http://www.vier.be/janigaat/videos/jani-gaat-naar-tokio-aflevering-4/2674839', |
80 | 'info_dict': { | |
81 | 'id': '2674839', | |
82 | 'display_id': 'jani-gaat-naar-tokio-aflevering-4', | |
83 | 'ext': 'mp4', | |
84 | 'title': 'jani-gaat-naar-tokio-aflevering-4', | |
85 | }, | |
86 | 'params': { | |
89fd0307 | 87 | 'skip_download': True, |
88 | }, | |
89 | 'expected_warnings': ['Log in to extract metadata'], | |
9eb4f404 | 90 | }, { |
7073015a S |
91 | # Without video id in URL |
92 | 'url': 'http://www.vier.be/planb/videos/dit-najaar-plan-b', | |
9eb4f404 S |
93 | 'only_matching': True, |
94 | }, { | |
95 | 'url': 'http://www.vier.be/video/v3/embed/16129', | |
96 | 'only_matching': True, | |
0a2e1b2e S |
97 | }, { |
98 | 'url': 'https://www.vijf.be/embed/video/public/4093', | |
99 | 'only_matching': True, | |
100 | }, { | |
101 | 'url': 'https://www.vier.be/video/blockbusters/in-juli-en-augustus-summer-classics', | |
102 | 'only_matching': True, | |
103 | }, { | |
104 | 'url': 'https://www.vier.be/video/achter-de-rug/2017/achter-de-rug-seizoen-1-aflevering-6', | |
105 | 'only_matching': True, | |
9eb4f404 S |
106 | }] |
107 | ||
89fd0307 | 108 | def _real_initialize(self): |
109 | self._logged_in = False | |
110 | ||
111 | def _login(self, site): | |
112 | username, password = self._get_login_info() | |
113 | if username is None or password is None: | |
114 | return | |
115 | ||
116 | login_page = self._download_webpage( | |
117 | 'http://www.%s.be/user/login' % site, | |
118 | None, note='Logging in', errnote='Unable to log in', | |
119 | data=urlencode_postdata({ | |
120 | 'form_id': 'user_login', | |
121 | 'name': username, | |
122 | 'pass': password, | |
123 | }), | |
124 | headers={'Content-Type': 'application/x-www-form-urlencoded'}) | |
125 | ||
126 | login_error = self._html_search_regex( | |
127 | r'(?s)<div class="messages error">\s*<div>\s*<h2.+?</h2>(.+?)<', | |
128 | login_page, 'login error', default=None) | |
129 | if login_error: | |
130 | self.report_warning('Unable to log in: %s' % login_error) | |
131 | else: | |
132 | self._logged_in = True | |
133 | ||
9eb4f404 | 134 | def _real_extract(self, url): |
5ad28e7f | 135 | mobj = self._match_valid_url(url) |
9eb4f404 S |
136 | embed_id = mobj.group('embed_id') |
137 | display_id = mobj.group('display_id') or embed_id | |
89fd0307 | 138 | video_id = mobj.group('id') or embed_id |
a3ba8a7a | 139 | site = mobj.group('site') |
9eb4f404 | 140 | |
89fd0307 | 141 | if not self._logged_in: |
142 | self._login(site) | |
143 | ||
9eb4f404 S |
144 | webpage = self._download_webpage(url, display_id) |
145 | ||
89fd0307 | 146 | if r'id="user-login"' in webpage: |
147 | self.report_warning( | |
148 | 'Log in to extract metadata', video_id=display_id) | |
149 | webpage = self._download_webpage( | |
150 | 'http://www.%s.be/video/v3/embed/%s' % (site, video_id), | |
151 | display_id) | |
152 | ||
9eb4f404 | 153 | video_id = self._search_regex( |
484c9d2d | 154 | [r'data-nid="(\d+)"', r'"nid"\s*:\s*"(\d+)"'], |
7073015a | 155 | webpage, 'video id', default=video_id or display_id) |
0a2e1b2e S |
156 | |
157 | playlist_url = self._search_regex( | |
158 | r'data-file=(["\'])(?P<url>(?:https?:)?//[^/]+/.+?\.m3u8.*?)\1', | |
159 | webpage, 'm3u8 url', default=None, group='url') | |
160 | ||
161 | if not playlist_url: | |
162 | application = self._search_regex( | |
163 | [r'data-application="([^"]+)"', r'"application"\s*:\s*"([^"]+)"'], | |
164 | webpage, 'application', default=site + '_vod') | |
165 | filename = self._search_regex( | |
166 | [r'data-filename="([^"]+)"', r'"filename"\s*:\s*"([^"]+)"'], | |
167 | webpage, 'filename') | |
168 | playlist_url = 'http://vod.streamcloud.be/%s/_definst_/mp4:%s.mp4/playlist.m3u8' % (application, filename) | |
169 | ||
650bd947 S |
170 | formats = self._extract_wowza_formats( |
171 | playlist_url, display_id, skip_protocols=['dash']) | |
19dbaeec | 172 | self._sort_formats(formats) |
9eb4f404 S |
173 | |
174 | title = self._og_search_title(webpage, default=display_id) | |
36b226d4 | 175 | description = self._html_search_regex( |
650bd947 S |
176 | r'(?s)<div\b[^>]+\bclass=(["\'])[^>]*?\bfield-type-text-with-summary\b[^>]*?\1[^>]*>.*?<p>(?P<value>.+?)</p>', |
177 | webpage, 'description', default=None, group='value') | |
178 | thumbnail = self._og_search_thumbnail(webpage, default=None) | |
36b226d4 | 179 | upload_date = unified_strdate(self._html_search_regex( |
650bd947 S |
180 | r'(?s)<div\b[^>]+\bclass=(["\'])[^>]*?\bfield-name-post-date\b[^>]*?\1[^>]*>.*?(?P<value>\d{2}/\d{2}/\d{4})', |
181 | webpage, 'upload date', default=None, group='value')) | |
182 | ||
183 | series = self._search_regex( | |
184 | r'data-program=(["\'])(?P<value>(?:(?!\1).)+)\1', webpage, | |
185 | 'series', default=None, group='value') | |
186 | episode_number = int_or_none(self._search_regex( | |
187 | r'(?i)aflevering (\d+)', title, 'episode number', default=None)) | |
188 | tags = re.findall(r'<a\b[^>]+\bhref=["\']/tags/[^>]+>([^<]+)<', webpage) | |
9eb4f404 S |
189 | |
190 | return { | |
191 | 'id': video_id, | |
192 | 'display_id': display_id, | |
193 | 'title': title, | |
194 | 'description': description, | |
195 | 'thumbnail': thumbnail, | |
650bd947 S |
196 | 'upload_date': upload_date, |
197 | 'series': series, | |
198 | 'episode_number': episode_number, | |
199 | 'tags': tags, | |
9eb4f404 | 200 | 'formats': formats, |
f58487b3 | 201 | } |
9eb4f404 S |
202 | |
203 | ||
204 | class VierVideosIE(InfoExtractor): | |
205 | IE_NAME = 'vier:videos' | |
a3ba8a7a | 206 | _VALID_URL = r'https?://(?:www\.)?(?P<site>vier|vijf)\.be/(?P<program>[^/]+)/videos(?:\?.*\bpage=(?P<page>\d+)|$)' |
9eb4f404 S |
207 | _TESTS = [{ |
208 | 'url': 'http://www.vier.be/demoestuin/videos', | |
209 | 'info_dict': { | |
210 | 'id': 'demoestuin', | |
211 | }, | |
212 | 'playlist_mincount': 153, | |
a3ba8a7a LV |
213 | }, { |
214 | 'url': 'http://www.vijf.be/temptationisland/videos', | |
215 | 'info_dict': { | |
216 | 'id': 'temptationisland', | |
217 | }, | |
218 | 'playlist_mincount': 159, | |
9eb4f404 S |
219 | }, { |
220 | 'url': 'http://www.vier.be/demoestuin/videos?page=6', | |
221 | 'info_dict': { | |
222 | 'id': 'demoestuin-page6', | |
223 | }, | |
224 | 'playlist_mincount': 20, | |
225 | }, { | |
226 | 'url': 'http://www.vier.be/demoestuin/videos?page=7', | |
227 | 'info_dict': { | |
228 | 'id': 'demoestuin-page7', | |
229 | }, | |
230 | 'playlist_mincount': 13, | |
231 | }] | |
232 | ||
233 | def _real_extract(self, url): | |
5ad28e7f | 234 | mobj = self._match_valid_url(url) |
9eb4f404 | 235 | program = mobj.group('program') |
a3ba8a7a | 236 | site = mobj.group('site') |
9eb4f404 | 237 | |
9eb4f404 S |
238 | page_id = mobj.group('page') |
239 | if page_id: | |
240 | page_id = int(page_id) | |
241 | start_page = page_id | |
9eb4f404 S |
242 | playlist_id = '%s-page%d' % (program, page_id) |
243 | else: | |
244 | start_page = 0 | |
9eb4f404 S |
245 | playlist_id = program |
246 | ||
247 | entries = [] | |
cc1ac110 | 248 | for current_page_id in itertools.count(start_page): |
9eb4f404 | 249 | current_page = self._download_webpage( |
a3ba8a7a | 250 | 'http://www.%s.be/%s/videos?page=%d' % (site, program, current_page_id), |
9eb4f404 | 251 | program, |
cc1ac110 | 252 | 'Downloading page %d' % (current_page_id + 1)) |
9eb4f404 | 253 | page_entries = [ |
a3ba8a7a | 254 | self.url_result('http://www.' + site + '.be' + video_url, 'Vier') |
9eb4f404 | 255 | for video_url in re.findall( |
a3ba8a7a | 256 | r'<h[23]><a href="(/[^/]+/videos/[^/]+(?:/\d+)?)">', current_page)] |
9eb4f404 | 257 | entries.extend(page_entries) |
cc1ac110 S |
258 | if page_id or '>Meer<' not in current_page: |
259 | break | |
9eb4f404 S |
260 | |
261 | return self.playlist_result(entries, playlist_id) |