]> jfr.im git - yt-dlp.git/blame - youtube_dl/extractor/vgtv.py
Add support for https for all extractors as preventive and future-proof measure
[yt-dlp.git] / youtube_dl / extractor / vgtv.py
CommitLineData
78149a96
MK
1# coding: utf-8
2from __future__ import unicode_literals
3
4import re
5
6from .common import InfoExtractor
d50116b8 7from .xstream import XstreamIE
4d454c5e
S
8from ..utils import (
9 ExtractorError,
10 float_or_none,
11)
78149a96 12
78149a96 13
d50116b8 14class VGTVIE(XstreamIE):
804afc58 15 IE_DESC = 'VGTV, BTTV, FTV, Aftenposten and Aftonbladet'
41c3b34b 16
17 _HOST_TO_APPNAME = {
18 'vgtv.no': 'vgtv',
19 'bt.no/tv': 'bttv',
20 'aftenbladet.no/tv': 'satv',
21 'fvn.no/fvntv': 'fvntv',
22 'aftenposten.no/webtv': 'aptv',
7177fd24 23 'ap.vgtv.no/webtv': 'aptv',
41c3b34b 24 }
25
26 _APP_NAME_TO_VENDOR = {
27 'vgtv': 'vgtv',
28 'bttv': 'bt',
29 'satv': 'sa',
30 'fvntv': 'fvn',
31 'aptv': 'ap',
32 }
33
0ceab847 34 _VALID_URL = r'''(?x)
41c3b34b 35 (?:https?://(?:www\.)?
804afc58 36 (?P<host>
41c3b34b 37 %s
0ceab847 38 )
7177fd24 39 /?
0ceab847 40 (?:
804afc58 41 \#!/(?:video|live)/|
42 embed?.*id=
41c3b34b 43 )|
44 (?P<appname>
45 %s
46 ):)
9f6b5176 47 (?P<id>\d+)
41c3b34b 48 ''' % ('|'.join(_HOST_TO_APPNAME.keys()), '|'.join(_APP_NAME_TO_VENDOR.keys()))
49
321c1e44
S
50 _TESTS = [
51 {
52 # streamType: vod
53 'url': 'http://www.vgtv.no/#!/video/84196/hevnen-er-soet-episode-10-abu',
54 'md5': 'b8be7a234cebb840c0d512c78013e02f',
55 'info_dict': {
56 'id': '84196',
57 'ext': 'mp4',
eecd6a46 58 'title': 'Hevnen er søt: Episode 10 - Abu',
321c1e44
S
59 'description': 'md5:e25e4badb5f544b04341e14abdc72234',
60 'thumbnail': 're:^https?://.*\.jpg',
61 'duration': 648.000,
62 'timestamp': 1404626400,
3fbeb95e
S
63 'upload_date': '20140706',
64 'view_count': int,
321c1e44
S
65 },
66 },
67 {
68 # streamType: wasLive
69 'url': 'http://www.vgtv.no/#!/live/100764/opptak-vgtv-foelger-em-kvalifiseringen',
70 'info_dict': {
71 'id': '100764',
b7bb0df2 72 'ext': 'flv',
321c1e44
S
73 'title': 'OPPTAK: VGTV følger EM-kvalifiseringen',
74 'description': 'md5:3772d9c0dc2dff92a886b60039a7d4d3',
75 'thumbnail': 're:^https?://.*\.jpg',
eecd6a46 76 'duration': 9103.0,
321c1e44 77 'timestamp': 1410113864,
3fbeb95e
S
78 'upload_date': '20140907',
79 'view_count': int,
321c1e44
S
80 },
81 'params': {
82 # m3u8 download
83 'skip_download': True,
84 },
00d24327 85 'skip': 'Video is no longer available',
321c1e44
S
86 },
87 {
00d24327 88 # streamType: wasLive
b4dd9835 89 'url': 'http://www.vgtv.no/#!/live/113063/direkte-v75-fra-solvalla',
321c1e44 90 'info_dict': {
b4dd9835 91 'id': '113063',
7c38af48 92 'ext': 'mp4',
00d24327 93 'title': 'V75 fra Solvalla 30.05.15',
b4dd9835 94 'description': 'md5:b3743425765355855f88e096acc93231',
321c1e44 95 'thumbnail': 're:^https?://.*\.jpg',
00d24327 96 'duration': 25966,
b4dd9835
S
97 'timestamp': 1432975582,
98 'upload_date': '20150530',
3fbeb95e 99 'view_count': int,
321c1e44 100 },
7c38af48
YCH
101 'params': {
102 # m3u8 download
103 'skip_download': True,
104 },
9f6b5176 105 },
106 {
00d24327 107 'url': 'http://www.aftenposten.no/webtv/#!/video/21039/trailer-sweatshop-i-can-t-take-any-more',
3b68efdc 108 'md5': 'fd828cd29774a729bf4d4425fe192972',
00d24327 109 'info_dict': {
110 'id': '21039',
7177fd24 111 'ext': 'mp4',
00d24327 112 'title': 'TRAILER: «SWEATSHOP» - I can´t take any more',
113 'description': 'md5:21891f2b0dd7ec2f78d84a50e54f8238',
114 'duration': 66,
115 'timestamp': 1417002452,
116 'upload_date': '20141126',
117 'view_count': int,
7177fd24 118 },
119 'params': {
120 # m3u8 download
121 'skip_download': True,
122 },
321c1e44 123 },
34e7dc81
S
124 {
125 'url': 'http://www.bt.no/tv/#!/video/100250/norling-dette-er-forskjellen-paa-1-divisjon-og-eliteserien',
126 'only_matching': True,
127 },
7177fd24 128 {
129 'url': 'http://ap.vgtv.no/webtv#!/video/111084/de-nye-bysyklene-lettere-bedre-gir-stoerre-hjul-og-feste-til-mobil',
130 'only_matching': True,
131 },
321c1e44 132 ]
78149a96 133
321c1e44 134 def _real_extract(self, url):
34e7dc81
S
135 mobj = re.match(self._VALID_URL, url)
136 video_id = mobj.group('id')
137 host = mobj.group('host')
41c3b34b 138 appname = self._HOST_TO_APPNAME[host] if host else mobj.group('appname')
139 vendor = self._APP_NAME_TO_VENDOR[appname]
34e7dc81 140
321c1e44 141 data = self._download_json(
34e7dc81 142 'http://svp.vg.no/svp/api/v1/%s/assets/%s?appName=%s-website'
41c3b34b 143 % (vendor, video_id, appname),
321c1e44 144 video_id, 'Downloading media JSON')
78149a96 145
4d454c5e
S
146 if data.get('status') == 'inactive':
147 raise ExtractorError(
148 'Video %s is no longer available' % video_id, expected=True)
149
d50116b8 150 info = {
151 'formats': [],
152 }
153 if len(video_id) == 5:
154 if appname == 'bttv':
155 info = self._extract_video_info('btno', video_id)
d50116b8 156
321c1e44 157 streams = data['streamUrls']
b4dd9835 158 stream_type = data.get('streamType')
78149a96 159
321c1e44 160 formats = []
78149a96 161
321c1e44
S
162 hls_url = streams.get('hls')
163 if hls_url:
7e5edcfd
S
164 formats.extend(self._extract_m3u8_formats(
165 hls_url, video_id, 'mp4', m3u8_id='hls', fatal=False))
78149a96 166
321c1e44 167 hds_url = streams.get('hds')
97f18fac 168 if hds_url:
169 hdcore_sign = 'hdcore=3.7.0'
41c3b34b 170 f4m_formats = self._extract_f4m_formats(
97f18fac 171 hds_url + '?%s' % hdcore_sign, video_id, f4m_id='hds', fatal=False)
9f6b5176 172 if f4m_formats:
97f18fac 173 for entry in f4m_formats:
174 # URLs without the extra param induce an 404 error
175 entry.update({'extra_param_to_segment_url': hdcore_sign})
176 formats.append(entry)
78149a96 177
9f6b5176 178 mp4_urls = streams.get('pseudostreaming') or []
321c1e44
S
179 mp4_url = streams.get('mp4')
180 if mp4_url:
9f6b5176 181 mp4_urls.append(mp4_url)
182 for mp4_url in mp4_urls:
183 format_info = {
184 'url': mp4_url,
9f6b5176 185 }
186 mobj = re.search('(\d+)_(\d+)_(\d+)', mp4_url)
187 if mobj:
3b68efdc 188 tbr = int(mobj.group(3))
9f6b5176 189 format_info.update({
190 'width': int(mobj.group(1)),
191 'height': int(mobj.group(2)),
3b68efdc 192 'tbr': tbr,
193 'format_id': 'mp4-%s' % tbr,
321c1e44 194 })
9f6b5176 195 formats.append(format_info)
196
d50116b8 197 info['formats'].extend(formats)
321c1e44 198
d50116b8 199 self._sort_formats(info['formats'])
200
201 info.update({
321c1e44 202 'id': video_id,
00d24327 203 'title': self._live_title(data['title']) if stream_type == 'live' else data['title'],
321c1e44
S
204 'description': data['description'],
205 'thumbnail': data['images']['main'] + '?t[]=900x506q80',
206 'timestamp': data['published'],
207 'duration': float_or_none(data['duration'], 1000),
208 'view_count': data['displays'],
b4dd9835 209 'is_live': True if stream_type == 'live' else False,
d50116b8 210 })
211 return info
0ceab847
S
212
213
214class BTArticleIE(InfoExtractor):
fe373287
S
215 IE_NAME = 'bt:article'
216 IE_DESC = 'Bergens Tidende Articles'
5886b38d 217 _VALID_URL = r'https?://(?:www\.)?bt\.no/(?:[^/]+/)+(?P<id>[^/]+)-\d+\.html'
0ceab847
S
218 _TEST = {
219 'url': 'http://www.bt.no/nyheter/lokalt/Kjemper-for-internatet-1788214.html',
3b68efdc 220 'md5': '2acbe8ad129b3469d5ae51b1158878df',
0ceab847
S
221 'info_dict': {
222 'id': '23199',
223 'ext': 'mp4',
224 'title': 'Alrekstad internat',
225 'description': 'md5:dc81a9056c874fedb62fc48a300dac58',
226 'thumbnail': 're:^https?://.*\.jpg',
227 'duration': 191,
228 'timestamp': 1289991323,
229 'upload_date': '20101117',
230 'view_count': int,
231 },
232 }
233
234 def _real_extract(self, url):
235 webpage = self._download_webpage(url, self._match_id(url))
236 video_id = self._search_regex(
3b68efdc 237 r'<video[^>]+data-id="(\d+)"', webpage, 'video id')
41c3b34b 238 return self.url_result('bttv:%s' % video_id, 'VGTV')
fe373287
S
239
240
241class BTVestlendingenIE(InfoExtractor):
242 IE_NAME = 'bt:vestlendingen'
243 IE_DESC = 'Bergens Tidende - Vestlendingen'
5886b38d 244 _VALID_URL = r'https?://(?:www\.)?bt\.no/spesial/vestlendingen/#!/(?P<id>\d+)'
3b68efdc 245 _TESTS = [{
fe373287
S
246 'url': 'http://www.bt.no/spesial/vestlendingen/#!/86588',
247 'md5': 'd7d17e3337dc80de6d3a540aefbe441b',
248 'info_dict': {
249 'id': '86588',
250 'ext': 'mov',
251 'title': 'Otto Wollertsen',
252 'description': 'Vestlendingen Otto Fredrik Wollertsen',
253 'timestamp': 1430473209,
254 'upload_date': '20150501',
255 },
3b68efdc 256 'skip': '404 Error',
257 }, {
258 'url': 'http://www.bt.no/spesial/vestlendingen/#!/86255',
259 'md5': 'a2893f8632e96389f4bdf36aa9463ceb',
260 'info_dict': {
261 'id': '86255',
262 'ext': 'mov',
263 'title': 'Du må tåle å fryse og være sulten',
264 'description': 'md5:b8046f4d022d5830ddab04865791d063',
265 'upload_date': '20150321',
266 'timestamp': 1426942023,
267 },
268 }]
fe373287
S
269
270 def _real_extract(self, url):
41c3b34b 271 return self.url_result('bttv:%s' % self._match_id(url), 'VGTV')