]>
Commit | Line | Data |
---|---|---|
2c3e0af9 RA |
1 | import re |
2 | ||
ae8d5a5c | 3 | from .adobepass import AdobePassIE |
2c3e0af9 | 4 | from ..utils import ( |
e897bd82 | 5 | ExtractorError, |
2c3e0af9 | 6 | determine_ext, |
e897bd82 | 7 | int_or_none, |
2c3e0af9 | 8 | parse_age_limit, |
3b4775e0 | 9 | remove_end, |
e897bd82 SS |
10 | remove_start, |
11 | traverse_obj, | |
1bdae7d3 | 12 | try_get, |
e2884db3 | 13 | unified_timestamp, |
e897bd82 | 14 | urlencode_postdata, |
2c3e0af9 RA |
15 | ) |
16 | ||
17 | ||
ae8d5a5c RA |
18 | class GoIE(AdobePassIE): |
19 | _SITE_INFO = { | |
20 | 'abc': { | |
21 | 'brand': '001', | |
22 | 'requestor_id': 'ABC', | |
23 | }, | |
24 | 'freeform': { | |
25 | 'brand': '002', | |
26 | 'requestor_id': 'ABCFamily', | |
27 | }, | |
28 | 'watchdisneychannel': { | |
29 | 'brand': '004', | |
118afcf5 | 30 | 'resource_id': 'Disney', |
ae8d5a5c RA |
31 | }, |
32 | 'watchdisneyjunior': { | |
33 | 'brand': '008', | |
118afcf5 | 34 | 'resource_id': 'DisneyJunior', |
ae8d5a5c RA |
35 | }, |
36 | 'watchdisneyxd': { | |
37 | 'brand': '009', | |
118afcf5 | 38 | 'resource_id': 'DisneyXD', |
a30c2f40 S |
39 | }, |
40 | 'disneynow': { | |
41 | 'brand': '011', | |
42 | 'resource_id': 'Disney', | |
29f7c58a | 43 | }, |
44 | 'fxnow.fxnetworks': { | |
45 | 'brand': '025', | |
46 | 'requestor_id': 'dtci', | |
47 | }, | |
2c3e0af9 | 48 | } |
aef9f87e S |
49 | _VALID_URL = r'''(?x) |
50 | https?:// | |
3b4775e0 | 51 | (?P<sub_domain> |
add96eb9 | 52 | (?:{}\.)?go|fxnow\.fxnetworks| |
3b4775e0 | 53 | (?:www\.)?(?:abc|freeform|disneynow) |
aef9f87e S |
54 | )\.com/ |
55 | (?: | |
56 | (?:[^/]+/)*(?P<id>[Vv][Dd][Kk][Aa]\w+)| | |
57 | (?:[^/]+/)*(?P<display_id>[^/?\#]+) | |
58 | ) | |
add96eb9 | 59 | '''.format(r'\.|'.join(list(_SITE_INFO.keys()))) |
2c3e0af9 | 60 | _TESTS = [{ |
bf2a5555 | 61 | 'url': 'http://abc.go.com/shows/designated-survivor/video/most-recent/VDKA3807643', |
2c3e0af9 | 62 | 'info_dict': { |
bf2a5555 | 63 | 'id': 'VDKA3807643', |
2c3e0af9 | 64 | 'ext': 'mp4', |
bf2a5555 RA |
65 | 'title': 'The Traitor in the White House', |
66 | 'description': 'md5:05b009d2d145a1e85d25111bd37222e8', | |
2c3e0af9 RA |
67 | }, |
68 | 'params': { | |
69 | # m3u8 download | |
70 | 'skip_download': True, | |
71 | }, | |
dd90a21c | 72 | 'skip': 'This content is no longer available.', |
2c3e0af9 | 73 | }, { |
e2884db3 | 74 | 'url': 'https://disneynow.com/shows/big-hero-6-the-series', |
bf2a5555 RA |
75 | 'info_dict': { |
76 | 'title': 'Doraemon', | |
77 | 'id': 'SH55574025', | |
78 | }, | |
79 | 'playlist_mincount': 51, | |
dd90a21c S |
80 | }, { |
81 | 'url': 'http://freeform.go.com/shows/shadowhunters/episodes/season-2/1-this-guilty-blood', | |
82 | 'info_dict': { | |
83 | 'id': 'VDKA3609139', | |
dd90a21c S |
84 | 'title': 'This Guilty Blood', |
85 | 'description': 'md5:f18e79ad1c613798d95fdabfe96cd292', | |
86 | 'age_limit': 14, | |
e2884db3 | 87 | 'episode': 'Episode 1', |
88 | 'upload_date': '20170102', | |
89 | 'season': 'Season 2', | |
90 | 'thumbnail': 'http://cdn1.edgedatg.com/aws/v2/abcf/Shadowhunters/video/201/ae5f75608d86bf88aa4f9f4aa76ab1b7/579x325-Q100_ae5f75608d86bf88aa4f9f4aa76ab1b7.jpg', | |
91 | 'duration': 2544, | |
92 | 'season_number': 2, | |
93 | 'series': 'Shadowhunters', | |
94 | 'episode_number': 1, | |
95 | 'timestamp': 1483387200, | |
add96eb9 | 96 | 'ext': 'mp4', |
dd90a21c S |
97 | }, |
98 | 'params': { | |
99 | 'geo_bypass_ip_block': '3.244.239.0/24', | |
100 | # m3u8 download | |
101 | 'skip_download': True, | |
102 | }, | |
103 | }, { | |
e2884db3 | 104 | 'url': 'https://abc.com/shows/the-rookie/episode-guide/season-04/12-the-knock', |
dd90a21c | 105 | 'info_dict': { |
e2884db3 | 106 | 'id': 'VDKA26050359', |
107 | 'title': 'The Knock', | |
108 | 'description': 'md5:0c2947e3ada4c31f28296db7db14aa64', | |
dd90a21c | 109 | 'age_limit': 14, |
e2884db3 | 110 | 'ext': 'mp4', |
111 | 'thumbnail': 'http://cdn1.edgedatg.com/aws/v2/abc/TheRookie/video/412/daf830d06e83b11eaf5c0a299d993ae3/1556x876-Q75_daf830d06e83b11eaf5c0a299d993ae3.jpg', | |
112 | 'episode': 'Episode 12', | |
113 | 'season_number': 4, | |
114 | 'season': 'Season 4', | |
115 | 'timestamp': 1642975200, | |
116 | 'episode_number': 12, | |
117 | 'upload_date': '20220123', | |
118 | 'series': 'The Rookie', | |
119 | 'duration': 2572, | |
dd90a21c S |
120 | }, |
121 | 'params': { | |
122 | 'geo_bypass_ip_block': '3.244.239.0/24', | |
123 | # m3u8 download | |
124 | 'skip_download': True, | |
125 | }, | |
29f7c58a | 126 | }, { |
127 | 'url': 'https://fxnow.fxnetworks.com/shows/better-things/video/vdka12782841', | |
128 | 'info_dict': { | |
129 | 'id': 'VDKA12782841', | |
29f7c58a | 130 | 'title': 'First Look: Better Things - Season 2', |
131 | 'description': 'md5:fa73584a95761c605d9d54904e35b407', | |
1bdae7d3 | 132 | 'ext': 'mp4', |
e2884db3 | 133 | 'age_limit': 14, |
134 | 'upload_date': '20170825', | |
135 | 'duration': 161, | |
136 | 'series': 'Better Things', | |
137 | 'thumbnail': 'http://cdn1.edgedatg.com/aws/v2/fx/BetterThings/video/12782841/b6b05e58264121cc2c98811318e6d507/1556x876-Q75_b6b05e58264121cc2c98811318e6d507.jpg', | |
138 | 'timestamp': 1503661074, | |
1bdae7d3 | 139 | }, |
140 | 'params': { | |
e2884db3 | 141 | 'geo_bypass_ip_block': '3.244.239.0/24', |
1bdae7d3 | 142 | # m3u8 download |
143 | 'skip_download': True, | |
144 | }, | |
692fa200 S |
145 | }, { |
146 | 'url': 'http://abc.go.com/shows/the-catch/episode-guide/season-01/10-the-wedding', | |
147 | 'only_matching': True, | |
148 | }, { | |
149 | 'url': 'http://abc.go.com/shows/world-news-tonight/episode-guide/2017-02/17-021717-intense-stand-off-between-man-with-rifle-and-police-in-oakland', | |
150 | 'only_matching': True, | |
52007de8 S |
151 | }, { |
152 | # brand 004 | |
153 | 'url': 'http://disneynow.go.com/shows/big-hero-6-the-series/season-01/episode-10-mr-sparkles-loses-his-sparkle/vdka4637915', | |
154 | 'only_matching': True, | |
155 | }, { | |
156 | # brand 008 | |
157 | 'url': 'http://disneynow.go.com/shows/minnies-bow-toons/video/happy-campers/vdka4872013', | |
158 | 'only_matching': True, | |
4f71473e S |
159 | }, { |
160 | 'url': 'https://disneynow.com/shows/minnies-bow-toons/video/happy-campers/vdka4872013', | |
161 | 'only_matching': True, | |
3b4775e0 | 162 | }, { |
163 | 'url': 'https://www.freeform.com/shows/cruel-summer/episode-guide/season-01/01-happy-birthday-jeanette-turner', | |
164 | 'only_matching': True, | |
2c3e0af9 RA |
165 | }] |
166 | ||
bf2a5555 RA |
167 | def _extract_videos(self, brand, video_id='-1', show_id='-1'): |
168 | display_id = video_id if video_id != '-1' else show_id | |
169 | return self._download_json( | |
add96eb9 | 170 | f'http://api.contents.watchabc.go.com/vp2/ws/contents/3000/videos/{brand}/001/-1/{show_id}/-1/{video_id}/-1/-1.json', |
bf2a5555 RA |
171 | display_id)['video'] |
172 | ||
2c3e0af9 | 173 | def _real_extract(self, url): |
5ad28e7f | 174 | mobj = self._match_valid_url(url) |
3b4775e0 | 175 | sub_domain = remove_start(remove_end(mobj.group('sub_domain') or '', '.go'), 'www.') |
a30c2f40 | 176 | video_id, display_id = mobj.group('id', 'display_id') |
52007de8 S |
177 | site_info = self._SITE_INFO.get(sub_domain, {}) |
178 | brand = site_info.get('brand') | |
179 | if not video_id or not site_info: | |
180 | webpage = self._download_webpage(url, display_id or video_id) | |
1bdae7d3 | 181 | data = self._parse_json( |
182 | self._search_regex( | |
183 | r'["\']__abc_com__["\']\s*\]\s*=\s*({.+?})\s*;', webpage, | |
184 | 'data', default='{}'), | |
185 | display_id or video_id, fatal=False) | |
186 | # https://abc.com/shows/modern-family/episode-guide/season-01/101-pilot | |
187 | layout = try_get(data, lambda x: x['page']['content']['video']['layout'], dict) | |
188 | video_id = None | |
189 | if layout: | |
190 | video_id = try_get( | |
191 | layout, | |
192 | (lambda x: x['videoid'], lambda x: x['video']['id']), | |
add96eb9 | 193 | str) |
1bdae7d3 | 194 | if not video_id: |
195 | video_id = self._search_regex( | |
196 | ( | |
197 | # There may be inner quotes, e.g. data-video-id="'VDKA3609139'" | |
198 | # from http://freeform.go.com/shows/shadowhunters/episodes/season-2/1-this-guilty-blood | |
199 | r'data-video-id=["\']*(VDKA\w+)', | |
200 | # page.analytics.videoIdCode | |
201 | r'\bvideoIdCode["\']\s*:\s*["\']((?:vdka|VDKA)\w+)', | |
202 | # https://abc.com/shows/the-rookie/episode-guide/season-02/03-the-bet | |
add96eb9 | 203 | r'\b(?:video)?id["\']\s*:\s*["\'](VDKA\w+)', |
1bdae7d3 | 204 | ), webpage, 'video id', default=video_id) |
52007de8 S |
205 | if not site_info: |
206 | brand = self._search_regex( | |
207 | (r'data-brand=\s*["\']\s*(\d+)', | |
208 | r'data-page-brand=\s*["\']\s*(\d+)'), webpage, 'brand', | |
209 | default='004') | |
210 | site_info = next( | |
211 | si for _, si in self._SITE_INFO.items() | |
212 | if si.get('brand') == brand) | |
bf2a5555 RA |
213 | if not video_id: |
214 | # show extraction works for Disney, DisneyJunior and DisneyXD | |
215 | # ABC and Freeform has different layout | |
216 | show_id = self._search_regex(r'data-show-id=["\']*(SH\d+)', webpage, 'show id') | |
217 | videos = self._extract_videos(brand, show_id=show_id) | |
218 | show_title = self._search_regex(r'data-show-title="([^"]+)"', webpage, 'show title', fatal=False) | |
219 | entries = [] | |
220 | for video in videos: | |
221 | entries.append(self.url_result( | |
222 | video['url'], 'Go', video.get('id'), video.get('title'))) | |
223 | entries.reverse() | |
224 | return self.playlist_result(entries, show_id, show_title) | |
225 | video_data = self._extract_videos(brand, video_id)[0] | |
226 | video_id = video_data['id'] | |
2c3e0af9 RA |
227 | title = video_data['title'] |
228 | ||
229 | formats = [] | |
47b8bf20 | 230 | subtitles = {} |
2c3e0af9 RA |
231 | for asset in video_data.get('assets', {}).get('asset', []): |
232 | asset_url = asset.get('value') | |
233 | if not asset_url: | |
234 | continue | |
235 | format_id = asset.get('format') | |
236 | ext = determine_ext(asset_url) | |
237 | if ext == 'm3u8': | |
014b7e6b | 238 | video_type = video_data.get('type') |
8e1409fd RA |
239 | data = { |
240 | 'video_id': video_data['id'], | |
241 | 'video_type': video_type, | |
242 | 'brand': brand, | |
243 | 'device': '001', | |
244 | } | |
245 | if video_data.get('accesslevel') == '1': | |
118afcf5 RA |
246 | requestor_id = site_info.get('requestor_id', 'DisneyChannels') |
247 | resource = site_info.get('resource_id') or self._get_mvpd_resource( | |
8e1409fd RA |
248 | requestor_id, title, video_id, None) |
249 | auth = self._extract_mvpd_auth( | |
250 | url, video_id, requestor_id, resource) | |
251 | data.update({ | |
252 | 'token': auth, | |
253 | 'token_type': 'ap', | |
254 | 'adobe_requestor_id': requestor_id, | |
255 | }) | |
256 | else: | |
5f95927a | 257 | self._initialize_geo_bypass({'countries': ['US']}) |
8e1409fd RA |
258 | entitlement = self._download_json( |
259 | 'https://api.entitlement.watchabc.go.com/vp2/ws-secure/entitlement/2020/authorize.json', | |
bf2a5555 | 260 | video_id, data=urlencode_postdata(data)) |
8e1409fd RA |
261 | errors = entitlement.get('errors', {}).get('errors', []) |
262 | if errors: | |
263 | for error in errors: | |
264 | if error.get('code') == 1002: | |
265 | self.raise_geo_restricted( | |
266 | error['message'], countries=['US']) | |
267 | error_message = ', '.join([error['message'] for error in errors]) | |
add96eb9 | 268 | raise ExtractorError(f'{self.IE_NAME} said: {error_message}', expected=True) |
8e1409fd | 269 | asset_url += '?' + entitlement['uplynkData']['sessionKey'] |
47b8bf20 F |
270 | fmts, subs = self._extract_m3u8_formats_and_subtitles( |
271 | asset_url, video_id, 'mp4', m3u8_id=format_id or 'hls', fatal=False) | |
272 | formats.extend(fmts) | |
273 | self._merge_subtitles(subs, target=subtitles) | |
2c3e0af9 | 274 | else: |
8e1409fd | 275 | f = { |
2c3e0af9 RA |
276 | 'format_id': format_id, |
277 | 'url': asset_url, | |
278 | 'ext': ext, | |
8e1409fd RA |
279 | } |
280 | if re.search(r'(?:/mp4/source/|_source\.mp4)', asset_url): | |
281 | f.update({ | |
add96eb9 | 282 | 'format_id': (f'{format_id}-' if format_id else '') + 'SOURCE', |
f983b875 | 283 | 'quality': 1, |
8e1409fd RA |
284 | }) |
285 | else: | |
286 | mobj = re.search(r'/(\d+)x(\d+)/', asset_url) | |
287 | if mobj: | |
288 | height = int(mobj.group(2)) | |
289 | f.update({ | |
add96eb9 | 290 | 'format_id': (f'{format_id}-' if format_id else '') + f'{height}P', |
8e1409fd RA |
291 | 'width': int(mobj.group(1)), |
292 | 'height': height, | |
293 | }) | |
294 | formats.append(f) | |
2c3e0af9 | 295 | |
2c3e0af9 RA |
296 | for cc in video_data.get('closedcaption', {}).get('src', []): |
297 | cc_url = cc.get('value') | |
298 | if not cc_url: | |
299 | continue | |
300 | ext = determine_ext(cc_url) | |
301 | if ext == 'xml': | |
302 | ext = 'ttml' | |
303 | subtitles.setdefault(cc.get('lang'), []).append({ | |
304 | 'url': cc_url, | |
305 | 'ext': ext, | |
306 | }) | |
307 | ||
308 | thumbnails = [] | |
309 | for thumbnail in video_data.get('thumbnails', {}).get('thumbnail', []): | |
310 | thumbnail_url = thumbnail.get('value') | |
311 | if not thumbnail_url: | |
312 | continue | |
313 | thumbnails.append({ | |
314 | 'url': thumbnail_url, | |
315 | 'width': int_or_none(thumbnail.get('width')), | |
316 | 'height': int_or_none(thumbnail.get('height')), | |
317 | }) | |
318 | ||
319 | return { | |
320 | 'id': video_id, | |
321 | 'title': title, | |
322 | 'description': video_data.get('longdescription') or video_data.get('description'), | |
323 | 'duration': int_or_none(video_data.get('duration', {}).get('value'), 1000), | |
324 | 'age_limit': parse_age_limit(video_data.get('tvrating', {}).get('rating')), | |
325 | 'episode_number': int_or_none(video_data.get('episodenumber')), | |
326 | 'series': video_data.get('show', {}).get('title'), | |
327 | 'season_number': int_or_none(video_data.get('season', {}).get('num')), | |
328 | 'thumbnails': thumbnails, | |
329 | 'formats': formats, | |
330 | 'subtitles': subtitles, | |
e2884db3 | 331 | 'timestamp': unified_timestamp(traverse_obj(video_data, ('airdates', 'airdate', 0))), |
2c3e0af9 | 332 | } |