]>
Commit | Line | Data |
---|---|---|
07256b9f S |
1 | import json |
2 | import uuid | |
4191779d | 3 | |
d664de44 | 4 | from .common import InfoExtractor |
f3c0c667 | 5 | from ..utils import ( |
07256b9f S |
6 | ExtractorError, |
7 | clean_html, | |
47da7823 | 8 | determine_ext, |
07256b9f | 9 | extract_attributes, |
f3c0c667 | 10 | float_or_none, |
07256b9f | 11 | get_elements_html_by_class, |
f3c0c667 | 12 | int_or_none, |
07256b9f | 13 | merge_dicts, |
4191779d | 14 | mimetype2ext, |
47da7823 | 15 | parse_iso8601, |
07256b9f | 16 | remove_end, |
47da7823 | 17 | remove_start, |
07256b9f S |
18 | str_or_none, |
19 | traverse_obj, | |
20 | url_or_none, | |
f3c0c667 | 21 | ) |
d664de44 S |
22 | |
23 | ||
50aa43b3 | 24 | class NYTimesBaseIE(InfoExtractor): |
07256b9f S |
25 | _DNS_NAMESPACE = uuid.UUID('36dd619a-56dc-595b-9e09-37f4152c7b5d') |
26 | _TOKEN = 'MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuNIzKBOFB77aT/jN/FQ+/QVKWq5V1ka1AYmCR9hstz1pGNPH5ajOU9gAqta0T89iPnhjwla+3oec/Z3kGjxbpv6miQXufHFq3u2RC6HyU458cLat5kVPSOQCe3VVB5NRpOlRuwKHqn0txfxnwSSj8mqzstR997d3gKB//RO9zE16y3PoWlDQXkASngNJEWvL19iob/xwAkfEWCjyRILWFY0JYX3AvLMSbq7wsqOCE5srJpo7rRU32zsByhsp1D5W9OYqqwDmflsgCEQy2vqTsJjrJohuNg+urMXNNZ7Y3naMoqttsGDrWVxtPBafKMI8pM2ReNZBbGQsQXRzQNo7+QIDAQAB' | |
27 | _GRAPHQL_API = 'https://samizdat-graphql.nytimes.com/graphql/v2' | |
28 | _GRAPHQL_QUERY = '''query VideoQuery($id: String!) { | |
29 | video(id: $id) { | |
30 | ... on Video { | |
31 | bylines { | |
32 | renderedRepresentation | |
33 | } | |
34 | duration | |
35 | promotionalHeadline | |
36 | promotionalMedia { | |
37 | ... on Image { | |
38 | crops { | |
39 | name | |
40 | renditions { | |
41 | name | |
42 | width | |
43 | height | |
44 | url | |
45 | } | |
46 | } | |
47 | } | |
48 | } | |
49 | renditions { | |
50 | type | |
51 | width | |
52 | height | |
53 | url | |
54 | bitrate | |
55 | } | |
56 | summary | |
57 | } | |
58 | } | |
59 | }''' | |
60 | ||
61 | def _call_api(self, media_id): | |
62 | # reference: `id-to-uri.js` | |
63 | video_uuid = uuid.uuid5(self._DNS_NAMESPACE, 'video') | |
64 | media_uuid = uuid.uuid5(video_uuid, media_id) | |
65 | ||
66 | return traverse_obj(self._download_json( | |
67 | self._GRAPHQL_API, media_id, 'Downloading JSON from GraphQL API', data=json.dumps({ | |
68 | 'query': self._GRAPHQL_QUERY, | |
69 | 'variables': {'id': f'nyt://video/{media_uuid}'}, | |
70 | }, separators=(',', ':')).encode(), headers={ | |
71 | 'Content-Type': 'application/json', | |
72 | 'Nyt-App-Type': 'vhs', | |
73 | 'Nyt-App-Version': 'v3.52.21', | |
74 | 'Nyt-Token': self._TOKEN, | |
75 | 'Origin': 'https://nytimes.com', | |
76 | }, fatal=False), ('data', 'video', {dict})) or {} | |
77 | ||
78 | def _extract_thumbnails(self, thumbs): | |
79 | return traverse_obj(thumbs, (lambda _, v: url_or_none(v['url']), { | |
80 | 'url': 'url', | |
81 | 'width': ('width', {int_or_none}), | |
82 | 'height': ('height', {int_or_none}), | |
83 | }), default=None) | |
84 | ||
85 | def _extract_formats_and_subtitles(self, video_id, content_media_json): | |
4191779d RA |
86 | urls = [] |
87 | formats = [] | |
47f4203d | 88 | subtitles = {} |
07256b9f | 89 | for video in traverse_obj(content_media_json, ('renditions', ..., {dict})): |
4191779d RA |
90 | video_url = video.get('url') |
91 | format_id = video.get('type') | |
92 | if not video_url or format_id == 'thumbs' or video_url in urls: | |
93 | continue | |
94 | urls.append(video_url) | |
95 | ext = mimetype2ext(video.get('mimetype')) or determine_ext(video_url) | |
96 | if ext == 'm3u8': | |
47f4203d | 97 | m3u8_fmts, m3u8_subs = self._extract_m3u8_formats_and_subtitles( |
4191779d | 98 | video_url, video_id, 'mp4', 'm3u8_native', |
47f4203d F |
99 | m3u8_id=format_id or 'hls', fatal=False) |
100 | formats.extend(m3u8_fmts) | |
07256b9f | 101 | self._merge_subtitles(m3u8_subs, target=subtitles) |
4191779d | 102 | elif ext == 'mpd': |
07256b9f | 103 | continue # all mpd urls give 404 errors |
4191779d RA |
104 | else: |
105 | formats.append({ | |
106 | 'url': video_url, | |
107 | 'format_id': format_id, | |
108 | 'vcodec': video.get('videoencoding') or video.get('video_codec'), | |
109 | 'width': int_or_none(video.get('width')), | |
110 | 'height': int_or_none(video.get('height')), | |
07256b9f S |
111 | 'filesize': traverse_obj(video, ( |
112 | ('file_size', 'fileSize'), (None, ('value')), {int_or_none}), get_all=False), | |
f377edec | 113 | 'tbr': int_or_none(video.get('bitrate'), 1000) or None, |
4191779d RA |
114 | 'ext': ext, |
115 | }) | |
d664de44 | 116 | |
07256b9f | 117 | return formats, subtitles |
4191779d | 118 | |
07256b9f S |
119 | def _extract_video(self, media_id): |
120 | data = self._call_api(media_id) | |
121 | formats, subtitles = self._extract_formats_and_subtitles(media_id, data) | |
d664de44 S |
122 | |
123 | return { | |
07256b9f S |
124 | 'id': media_id, |
125 | 'title': data.get('promotionalHeadline'), | |
126 | 'description': data.get('summary'), | |
127 | 'duration': float_or_none(data.get('duration'), scale=1000), | |
128 | 'creator': ', '.join(traverse_obj(data, ( # TODO: change to 'creators' | |
129 | 'bylines', ..., 'renderedRepresentation', {lambda x: remove_start(x, 'By ')}))), | |
d664de44 | 130 | 'formats': formats, |
47f4203d | 131 | 'subtitles': subtitles, |
07256b9f S |
132 | 'thumbnails': self._extract_thumbnails( |
133 | traverse_obj(data, ('promotionalMedia', 'crops', ..., 'renditions', ...))), | |
5f6a1245 | 134 | } |
50aa43b3 YCH |
135 | |
136 | ||
137 | class NYTimesIE(NYTimesBaseIE): | |
138 | _VALID_URL = r'https?://(?:(?:www\.)?nytimes\.com/video/(?:[^/]+/)+?|graphics8\.nytimes\.com/bcvideo/\d+(?:\.\d+)?/iframe/embed\.html\?videoId=)(?P<id>\d+)' | |
bfd973ec | 139 | _EMBED_REGEX = [r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//graphics8\.nytimes\.com/bcvideo/[^/]+/iframe/embed\.html.+?)\1>'] |
50aa43b3 YCH |
140 | _TESTS = [{ |
141 | 'url': 'http://www.nytimes.com/video/opinion/100000002847155/verbatim-what-is-a-photocopier.html?playlistId=100000001150263', | |
07256b9f | 142 | 'md5': 'a553aa344014e3723d33893d89d4defc', |
50aa43b3 YCH |
143 | 'info_dict': { |
144 | 'id': '100000002847155', | |
07256b9f | 145 | 'ext': 'mp4', |
50aa43b3 YCH |
146 | 'title': 'Verbatim: What Is a Photocopier?', |
147 | 'description': 'md5:93603dada88ddbda9395632fdc5da260', | |
07256b9f S |
148 | 'timestamp': 1398631707, # FIXME |
149 | 'upload_date': '20140427', # FIXME | |
150 | 'creator': 'Brett Weiner', | |
151 | 'thumbnail': r're:https?://\w+\.nyt.com/images/.+\.jpg', | |
50aa43b3 | 152 | 'duration': 419, |
07256b9f | 153 | }, |
50aa43b3 YCH |
154 | }, { |
155 | 'url': 'http://www.nytimes.com/video/travel/100000003550828/36-hours-in-dubai.html', | |
156 | 'only_matching': True, | |
157 | }] | |
158 | ||
159 | def _real_extract(self, url): | |
160 | video_id = self._match_id(url) | |
161 | ||
07256b9f | 162 | return self._extract_video(video_id) |
50aa43b3 YCH |
163 | |
164 | ||
165 | class NYTimesArticleIE(NYTimesBaseIE): | |
07256b9f | 166 | _VALID_URL = r'https?://(?:www\.)?nytimes\.com/\d{4}/\d{2}/\d{2}/(?!books|podcasts)[^/?#]+/(?:\w+/)?(?P<id>[^./?#]+)(?:\.html)?' |
df8418ff | 167 | _TESTS = [{ |
50aa43b3 | 168 | 'url': 'http://www.nytimes.com/2015/04/14/business/owner-of-gravity-payments-a-credit-card-processor-is-setting-a-new-minimum-wage-70000-a-year.html?_r=0', |
07256b9f | 169 | 'md5': '3eb5ddb1d6f86254fe4f233826778737', |
50aa43b3 YCH |
170 | 'info_dict': { |
171 | 'id': '100000003628438', | |
07256b9f S |
172 | 'ext': 'mp4', |
173 | 'title': 'One Company’s New Minimum Wage: $70,000 a Year', | |
174 | 'description': 'md5:89ba9ab67ca767bb92bf823d1f138433', | |
175 | 'timestamp': 1429047468, | |
50aa43b3 YCH |
176 | 'upload_date': '20150414', |
177 | 'uploader': 'Matthew Williams', | |
07256b9f S |
178 | 'creator': 'Patricia Cohen', |
179 | 'thumbnail': r're:https?://\w+\.nyt.com/images/.*\.jpg', | |
180 | 'duration': 119.0, | |
181 | }, | |
74324a7a | 182 | }, { |
07256b9f S |
183 | # article with audio and no video |
184 | 'url': 'https://www.nytimes.com/2023/09/29/health/mosquitoes-genetic-engineering.html', | |
185 | 'md5': '2365b3555c8aa7f4dd34ca735ad02e6a', | |
74324a7a | 186 | 'info_dict': { |
07256b9f | 187 | 'id': '100000009110381', |
74324a7a | 188 | 'ext': 'mp3', |
07256b9f S |
189 | 'title': 'The Gamble: Can Genetically Modified Mosquitoes End Disease?', |
190 | 'description': 'md5:9ff8b47acbaf7f3ca8c732f5c815be2e', | |
191 | 'timestamp': 1695960700, | |
192 | 'upload_date': '20230929', | |
193 | 'creator': 'Stephanie Nolen, Natalija Gormalova', | |
194 | 'thumbnail': r're:https?://\w+\.nyt.com/images/.*\.jpg', | |
195 | 'duration': 1322, | |
196 | }, | |
74324a7a | 197 | }, { |
07256b9f S |
198 | 'url': 'https://www.nytimes.com/2023/11/29/business/dealbook/kamala-harris-biden-voters.html', |
199 | 'md5': '3eb5ddb1d6f86254fe4f233826778737', | |
74324a7a | 200 | 'info_dict': { |
07256b9f S |
201 | 'id': '100000009202270', |
202 | 'ext': 'mp4', | |
203 | 'title': 'Kamala Harris Defends Biden Policies, but Says ‘More Work’ Needed to Reach Voters', | |
204 | 'description': 'md5:de4212a7e19bb89e4fb14210ca915f1f', | |
205 | 'timestamp': 1701290997, | |
206 | 'upload_date': '20231129', | |
207 | 'uploader': 'By The New York Times', | |
208 | 'creator': 'Katie Rogers', | |
209 | 'thumbnail': r're:https?://\w+\.nyt.com/images/.*\.jpg', | |
210 | 'duration': 97.631, | |
47da7823 S |
211 | }, |
212 | 'params': { | |
07256b9f S |
213 | 'skip_download': 'm3u8', |
214 | }, | |
215 | }, { | |
216 | # multiple videos in the same article | |
217 | 'url': 'https://www.nytimes.com/2023/12/02/business/air-traffic-controllers-safety.html', | |
218 | 'info_dict': { | |
219 | 'id': 'air-traffic-controllers-safety', | |
220 | 'title': 'Drunk and Asleep on the Job: Air Traffic Controllers Pushed to the Brink', | |
221 | 'description': 'md5:549e5a5e935bf7d048be53ba3d2c863d', | |
222 | 'upload_date': '20231202', | |
223 | 'creator': 'Emily Steel, Sydney Ember', | |
224 | 'timestamp': 1701511264, | |
47da7823 | 225 | }, |
07256b9f | 226 | 'playlist_count': 3, |
df8418ff | 227 | }, { |
07256b9f | 228 | 'url': 'https://www.nytimes.com/2023/12/02/business/media/netflix-squid-game-challenge.html', |
df8418ff YCH |
229 | 'only_matching': True, |
230 | }] | |
50aa43b3 | 231 | |
07256b9f S |
232 | def _extract_content_from_block(self, block): |
233 | details = traverse_obj(block, { | |
234 | 'id': ('sourceId', {str}), | |
235 | 'uploader': ('bylines', ..., 'renderedRepresentation', {str}), | |
236 | 'duration': (None, (('duration', {lambda x: float_or_none(x, scale=1000)}), ('length', {int_or_none}))), | |
237 | 'timestamp': ('firstPublished', {parse_iso8601}), | |
238 | 'series': ('podcastSeries', {str}), | |
239 | }, get_all=False) | |
240 | ||
241 | formats, subtitles = self._extract_formats_and_subtitles(details.get('id'), block) | |
242 | # audio articles will have an url and no formats | |
243 | url = traverse_obj(block, ('fileUrl', {url_or_none})) | |
244 | if not formats and url: | |
245 | formats.append({'url': url, 'vcodec': 'none'}) | |
47da7823 | 246 | |
07256b9f S |
247 | return { |
248 | **details, | |
249 | 'thumbnails': self._extract_thumbnails(traverse_obj( | |
250 | block, ('promotionalMedia', 'crops', ..., 'renditions', ...))), | |
251 | 'formats': formats, | |
252 | 'subtitles': subtitles | |
253 | } | |
47da7823 | 254 | |
07256b9f S |
255 | def _real_extract(self, url): |
256 | page_id = self._match_id(url) | |
257 | webpage = self._download_webpage(url, page_id) | |
258 | art_json = self._search_json( | |
259 | r'window\.__preloadedData\s*=', webpage, 'media details', page_id, | |
260 | transform_source=lambda x: x.replace('undefined', 'null'))['initialData']['data']['article'] | |
261 | ||
262 | blocks = traverse_obj(art_json, ( | |
263 | 'sprinkledBody', 'content', ..., ('ledeMedia', None), | |
264 | lambda _, v: v['__typename'] in ('Video', 'Audio'))) | |
265 | if not blocks: | |
266 | raise ExtractorError('Unable to extract any media blocks from webpage') | |
267 | ||
268 | common_info = { | |
269 | 'title': remove_end(self._html_extract_title(webpage), ' - The New York Times'), | |
270 | 'description': traverse_obj(art_json, ( | |
271 | 'sprinkledBody', 'content', ..., 'summary', 'content', ..., 'text', {str}), | |
272 | get_all=False) or self._html_search_meta(['og:description', 'twitter:description'], webpage), | |
273 | 'timestamp': traverse_obj(art_json, ('firstPublished', {parse_iso8601})), | |
274 | 'creator': ', '.join( | |
275 | traverse_obj(art_json, ('bylines', ..., 'creators', ..., 'displayName'))), # TODO: change to 'creators' (list) | |
276 | 'thumbnails': self._extract_thumbnails(traverse_obj( | |
277 | art_json, ('promotionalMedia', 'assetCrops', ..., 'renditions', ...))), | |
278 | } | |
47da7823 | 279 | |
07256b9f S |
280 | entries = [] |
281 | for block in blocks: | |
282 | entries.append(merge_dicts(self._extract_content_from_block(block), common_info)) | |
47da7823 | 283 | |
07256b9f S |
284 | if len(entries) > 1: |
285 | return self.playlist_result(entries, page_id, **common_info) | |
47da7823 S |
286 | |
287 | return { | |
07256b9f S |
288 | 'id': page_id, |
289 | **entries[0], | |
47da7823 S |
290 | } |
291 | ||
07256b9f S |
292 | |
293 | class NYTimesCookingIE(NYTimesBaseIE): | |
294 | IE_NAME = 'NYTimesCookingGuide' | |
295 | _VALID_URL = r'https?://cooking\.nytimes\.com/guides/(?P<id>[\w-]+)' | |
296 | _TESTS = [{ | |
297 | 'url': 'https://cooking.nytimes.com/guides/13-how-to-cook-a-turkey', | |
298 | 'info_dict': { | |
299 | 'id': '13-how-to-cook-a-turkey', | |
300 | 'title': 'How to Cook a Turkey', | |
301 | 'description': 'md5:726cfd3f9b161bdf5c279879e8050ca0', | |
302 | }, | |
303 | 'playlist_count': 2, | |
304 | }, { | |
305 | # single video example | |
306 | 'url': 'https://cooking.nytimes.com/guides/50-how-to-make-mac-and-cheese', | |
307 | 'md5': '64415805fe0b8640fce6b0b9def5989a', | |
308 | 'info_dict': { | |
309 | 'id': '100000005835845', | |
310 | 'ext': 'mp4', | |
311 | 'title': 'How to Make Mac and Cheese', | |
312 | 'description': 'md5:b8f2f33ec1fb7523b21367147c9594f1', | |
313 | 'duration': 9.51, | |
314 | 'creator': 'Alison Roman', | |
315 | 'thumbnail': r're:https?://\w+\.nyt.com/images/.*\.jpg', | |
316 | }, | |
317 | }, { | |
318 | 'url': 'https://cooking.nytimes.com/guides/20-how-to-frost-a-cake', | |
319 | 'md5': '64415805fe0b8640fce6b0b9def5989a', | |
320 | 'info_dict': { | |
321 | 'id': '20-how-to-frost-a-cake', | |
322 | 'title': 'How to Frost a Cake', | |
323 | 'description': 'md5:a31fe3b98a8ce7b98aae097730c269cd', | |
324 | }, | |
325 | 'playlist_count': 8, | |
326 | }] | |
327 | ||
50aa43b3 | 328 | def _real_extract(self, url): |
74324a7a | 329 | page_id = self._match_id(url) |
74324a7a | 330 | webpage = self._download_webpage(url, page_id) |
07256b9f S |
331 | title = self._html_search_meta(['og:title', 'twitter:title'], webpage) |
332 | description = self._html_search_meta(['og:description', 'twitter:description'], webpage) | |
50aa43b3 | 333 | |
07256b9f S |
334 | lead_video_id = self._search_regex( |
335 | r'data-video-player-id="(\d+)"></div>', webpage, 'lead video') | |
336 | media_ids = traverse_obj( | |
337 | get_elements_html_by_class('video-item', webpage), (..., {extract_attributes}, 'data-video-id')) | |
47da7823 | 338 | |
07256b9f S |
339 | if media_ids: |
340 | media_ids.append(lead_video_id) | |
341 | return self.playlist_result( | |
342 | [self._extract_video(media_id) for media_id in media_ids], page_id, title, description) | |
70c5802b | 343 | |
07256b9f S |
344 | return { |
345 | **self._extract_video(lead_video_id), | |
346 | 'title': title, | |
347 | 'description': description, | |
348 | 'creator': self._search_regex( # TODO: change to 'creators' | |
349 | r'<span itemprop="author">([^<]+)</span></p>', webpage, 'author', default=None), | |
350 | } | |
70c5802b | 351 | |
07256b9f S |
352 | |
353 | class NYTimesCookingRecipeIE(InfoExtractor): | |
354 | _VALID_URL = r'https?://cooking\.nytimes\.com/recipes/(?P<id>\d+)' | |
70c5802b | 355 | _TESTS = [{ |
356 | 'url': 'https://cooking.nytimes.com/recipes/1017817-cranberry-curd-tart', | |
07256b9f | 357 | 'md5': '579e83bbe8e61e9de67f80edba8a78a8', |
70c5802b | 358 | 'info_dict': { |
07256b9f S |
359 | 'id': '1017817', |
360 | 'ext': 'mp4', | |
361 | 'title': 'Cranberry Curd Tart', | |
362 | 'description': 'md5:ad77a3fc321db636256d4343c5742152', | |
363 | 'timestamp': 1447804800, | |
364 | 'upload_date': '20151118', | |
365 | 'creator': 'David Tanis', | |
366 | 'thumbnail': r're:https?://\w+\.nyt.com/images/.*\.jpg', | |
70c5802b | 367 | }, |
368 | }, { | |
07256b9f S |
369 | 'url': 'https://cooking.nytimes.com/recipes/1024781-neapolitan-checkerboard-cookies', |
370 | 'md5': '58df35998241dcf0620e99e646331b42', | |
70c5802b | 371 | 'info_dict': { |
07256b9f S |
372 | 'id': '1024781', |
373 | 'ext': 'mp4', | |
374 | 'title': 'Neapolitan Checkerboard Cookies', | |
375 | 'description': 'md5:ba12394c585ababea951cb6d2fcc6631', | |
376 | 'timestamp': 1701302400, | |
377 | 'upload_date': '20231130', | |
378 | 'creator': 'Sue Li', | |
379 | 'thumbnail': r're:https?://\w+\.nyt.com/images/.*\.jpg', | |
380 | }, | |
381 | }, { | |
382 | 'url': 'https://cooking.nytimes.com/recipes/1019516-overnight-oats', | |
383 | 'md5': '2fe7965a3adc899913b8e25ada360823', | |
384 | 'info_dict': { | |
385 | 'id': '1019516', | |
386 | 'ext': 'mp4', | |
387 | 'timestamp': 1546387200, | |
388 | 'description': 'md5:8856ce10239161bd2596ac335b9f9bfb', | |
389 | 'upload_date': '20190102', | |
390 | 'title': 'Overnight Oats', | |
391 | 'creator': 'Genevieve Ko', | |
392 | 'thumbnail': r're:https?://\w+\.nyt.com/images/.*\.jpg', | |
393 | }, | |
70c5802b | 394 | }] |
395 | ||
396 | def _real_extract(self, url): | |
397 | page_id = self._match_id(url) | |
70c5802b | 398 | webpage = self._download_webpage(url, page_id) |
07256b9f | 399 | recipe_data = self._search_nextjs_data(webpage, page_id)['props']['pageProps']['recipe'] |
70c5802b | 400 | |
07256b9f S |
401 | formats, subtitles = self._extract_m3u8_formats_and_subtitles( |
402 | recipe_data['videoSrc'], page_id, 'mp4', m3u8_id='hls') | |
70c5802b | 403 | |
07256b9f S |
404 | return { |
405 | **traverse_obj(recipe_data, { | |
406 | 'id': ('id', {str_or_none}), | |
407 | 'title': ('title', {str}), | |
408 | 'description': ('topnote', {clean_html}), | |
409 | 'timestamp': ('publishedAt', {int_or_none}), | |
410 | 'creator': ('contentAttribution', 'cardByline', {str}), | |
411 | }), | |
412 | 'formats': formats, | |
413 | 'subtitles': subtitles, | |
414 | 'thumbnails': [{'url': thumb_url} for thumb_url in traverse_obj( | |
415 | recipe_data, ('image', 'crops', 'recipe', ..., {url_or_none}))], | |
416 | } |