]>
Commit | Line | Data |
---|---|---|
0de668af JMF |
1 | from __future__ import unicode_literals |
2 | ||
cba5d1b6 | 3 | import itertools |
315ab3d5 | 4 | import hashlib |
27b1c73f | 5 | import json |
59fc531f JMF |
6 | import re |
7 | ||
8 | from .common import InfoExtractor | |
238d42cf S |
9 | from ..compat import ( |
10 | compat_str, | |
11 | compat_HTTPError, | |
12 | ) | |
e1ec9330 | 13 | from ..utils import ( |
238d42cf | 14 | ExtractorError, |
cce889b9 | 15 | float_or_none, |
c4096e8a | 16 | get_element_by_attribute, |
e1ec9330 | 17 | int_or_none, |
87696e78 | 18 | lowercase_escape, |
238d42cf | 19 | std_headers, |
98960c91 | 20 | try_get, |
3052a30d | 21 | url_or_none, |
6606817a | 22 | variadic, |
e1ec9330 | 23 | ) |
59fc531f | 24 | |
0de668af | 25 | |
59fc531f | 26 | class InstagramIE(InfoExtractor): |
29f7c58a | 27 | _VALID_URL = r'(?P<url>https?://(?:www\.)?instagram\.com/(?:p|tv|reel)/(?P<id>[^/?#&]+))' |
4479600d | 28 | _TESTS = [{ |
fc6e75dd | 29 | 'url': 'https://instagram.com/p/aye83DjauH/?foo=bar#abc', |
0de668af JMF |
30 | 'md5': '0d2da106a9d2631273e192b372806516', |
31 | 'info_dict': { | |
32 | 'id': 'aye83DjauH', | |
33 | 'ext': 'mp4', | |
0de668af JMF |
34 | 'title': 'Video by naomipq', |
35 | 'description': 'md5:1f17f0ab29bd6fe2bfad705f58de3cb8', | |
ec85ded8 | 36 | 'thumbnail': r're:^https?://.*\.jpg', |
cce889b9 | 37 | 'duration': 0, |
98960c91 S |
38 | 'timestamp': 1371748545, |
39 | 'upload_date': '20130620', | |
40 | 'uploader_id': 'naomipq', | |
29f7c58a | 41 | 'uploader': 'B E A U T Y F O R A S H E S', |
98960c91 S |
42 | 'like_count': int, |
43 | 'comment_count': int, | |
a56e74e2 | 44 | 'comments': list, |
98960c91 | 45 | }, |
fb4b3458 S |
46 | }, { |
47 | # missing description | |
48 | 'url': 'https://www.instagram.com/p/BA-pQFBG8HZ/?taken-by=britneyspears', | |
49 | 'info_dict': { | |
50 | 'id': 'BA-pQFBG8HZ', | |
51 | 'ext': 'mp4', | |
fb4b3458 | 52 | 'title': 'Video by britneyspears', |
ec85ded8 | 53 | 'thumbnail': r're:^https?://.*\.jpg', |
cce889b9 | 54 | 'duration': 0, |
98960c91 S |
55 | 'timestamp': 1453760977, |
56 | 'upload_date': '20160125', | |
57 | 'uploader_id': 'britneyspears', | |
58 | 'uploader': 'Britney Spears', | |
59 | 'like_count': int, | |
60 | 'comment_count': int, | |
a56e74e2 | 61 | 'comments': list, |
fb4b3458 S |
62 | }, |
63 | 'params': { | |
64 | 'skip_download': True, | |
65 | }, | |
ada77fa5 S |
66 | }, { |
67 | # multi video post | |
68 | 'url': 'https://www.instagram.com/p/BQ0eAlwhDrw/', | |
69 | 'playlist': [{ | |
70 | 'info_dict': { | |
71 | 'id': 'BQ0dSaohpPW', | |
72 | 'ext': 'mp4', | |
73 | 'title': 'Video 1', | |
74 | }, | |
75 | }, { | |
76 | 'info_dict': { | |
77 | 'id': 'BQ0dTpOhuHT', | |
78 | 'ext': 'mp4', | |
79 | 'title': 'Video 2', | |
80 | }, | |
81 | }, { | |
82 | 'info_dict': { | |
83 | 'id': 'BQ0dT7RBFeF', | |
84 | 'ext': 'mp4', | |
85 | 'title': 'Video 3', | |
86 | }, | |
87 | }], | |
88 | 'info_dict': { | |
89 | 'id': 'BQ0eAlwhDrw', | |
90 | 'title': 'Post by instagram', | |
91 | 'description': 'md5:0f9203fc6a2ce4d228da5754bcf54957', | |
92 | }, | |
cce889b9 | 93 | }, { |
94 | # IGTV | |
95 | 'url': 'https://www.instagram.com/tv/BkfuX9UB-eK/', | |
96 | 'info_dict': { | |
97 | 'id': 'BkfuX9UB-eK', | |
98 | 'ext': 'mp4', | |
99 | 'title': 'Fingerboarding Tricks with @cass.fb', | |
100 | 'thumbnail': r're:^https?://.*\.jpg', | |
101 | 'duration': 53.83, | |
102 | 'timestamp': 1530032919, | |
103 | 'upload_date': '20180626', | |
104 | 'uploader_id': 'instagram', | |
105 | 'uploader': 'Instagram', | |
106 | 'like_count': int, | |
107 | 'comment_count': int, | |
108 | 'comments': list, | |
109 | 'description': 'Meet Cass Hirst (@cass.fb), a fingerboarding pro who can perform tiny ollies and kickflips while blindfolded.', | |
110 | } | |
4479600d S |
111 | }, { |
112 | 'url': 'https://instagram.com/p/-Cmh1cukG2/', | |
113 | 'only_matching': True, | |
0dafea02 S |
114 | }, { |
115 | 'url': 'http://instagram.com/p/9o6LshA7zy/embed/', | |
116 | 'only_matching': True, | |
edb2820c RA |
117 | }, { |
118 | 'url': 'https://www.instagram.com/tv/aye83DjauH/', | |
119 | 'only_matching': True, | |
29f7c58a | 120 | }, { |
121 | 'url': 'https://www.instagram.com/reel/CDUMkliABpa/', | |
122 | 'only_matching': True, | |
4479600d | 123 | }] |
59fc531f | 124 | |
c4096e8a YCH |
125 | @staticmethod |
126 | def _extract_embed_url(webpage): | |
c23533a1 S |
127 | mobj = re.search( |
128 | r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?instagram\.com/p/[^/]+/embed.*?)\1', | |
129 | webpage) | |
130 | if mobj: | |
131 | return mobj.group('url') | |
132 | ||
c4096e8a YCH |
133 | blockquote_el = get_element_by_attribute( |
134 | 'class', 'instagram-media', webpage) | |
135 | if blockquote_el is None: | |
136 | return | |
137 | ||
138 | mobj = re.search( | |
139 | r'<a[^>]+href=([\'"])(?P<link>[^\'"]+)\1', blockquote_el) | |
140 | if mobj: | |
141 | return mobj.group('link') | |
142 | ||
59fc531f | 143 | def _real_extract(self, url): |
5ad28e7f | 144 | mobj = self._match_valid_url(url) |
0dafea02 S |
145 | video_id = mobj.group('id') |
146 | url = mobj.group('url') | |
d2d8248f | 147 | |
a0c716bb | 148 | webpage, urlh = self._download_webpage_handle(url, video_id) |
149 | if 'www.instagram.com/accounts/login' in urlh.geturl().rstrip('/'): | |
150 | self.raise_login_required('You need to log in to access this content', method='cookies') | |
98960c91 | 151 | |
29f7c58a | 152 | (media, video_url, description, thumbnail, timestamp, uploader, |
18848d22 | 153 | uploader_id, like_count, comment_count, comments, height, |
29f7c58a | 154 | width) = [None] * 12 |
155 | ||
156 | shared_data = self._parse_json( | |
157 | self._search_regex( | |
158 | r'window\._sharedData\s*=\s*({.+?});', | |
159 | webpage, 'shared data', default='{}'), | |
160 | video_id, fatal=False) | |
98960c91 S |
161 | if shared_data: |
162 | media = try_get( | |
18848d22 | 163 | shared_data, |
29f7c58a | 164 | (lambda x: x['entry_data']['PostPage'][0]['graphql']['shortcode_media'], |
165 | lambda x: x['entry_data']['PostPage'][0]['media']), | |
18848d22 | 166 | dict) |
29f7c58a | 167 | # _sharedData.entry_data.PostPage is empty when authenticated (see |
168 | # https://github.com/ytdl-org/youtube-dl/pull/22880) | |
169 | if not media: | |
170 | additional_data = self._parse_json( | |
171 | self._search_regex( | |
172 | r'window\.__additionalDataLoaded\s*\(\s*[^,]+,\s*({.+?})\s*\)\s*;', | |
173 | webpage, 'additional data', default='{}'), | |
174 | video_id, fatal=False) | |
175 | if additional_data: | |
176 | media = try_get( | |
177 | additional_data, lambda x: x['graphql']['shortcode_media'], | |
178 | dict) | |
179 | if media: | |
180 | video_url = media.get('video_url') | |
181 | height = int_or_none(media.get('dimensions', {}).get('height')) | |
182 | width = int_or_none(media.get('dimensions', {}).get('width')) | |
183 | description = try_get( | |
184 | media, lambda x: x['edge_media_to_caption']['edges'][0]['node']['text'], | |
185 | compat_str) or media.get('caption') | |
cce889b9 | 186 | title = media.get('title') |
29f7c58a | 187 | thumbnail = media.get('display_src') or media.get('display_url') |
cce889b9 | 188 | duration = float_or_none(media.get('video_duration')) |
29f7c58a | 189 | timestamp = int_or_none(media.get('taken_at_timestamp') or media.get('date')) |
190 | uploader = media.get('owner', {}).get('full_name') | |
191 | uploader_id = media.get('owner', {}).get('username') | |
192 | ||
193 | def get_count(keys, kind): | |
6606817a | 194 | for key in variadic(keys): |
29f7c58a | 195 | count = int_or_none(try_get( |
9cbd4dda S |
196 | media, (lambda x: x['edge_media_%s' % key]['count'], |
197 | lambda x: x['%ss' % kind]['count']))) | |
29f7c58a | 198 | if count is not None: |
199 | return count | |
60c8fc73 | 200 | |
29f7c58a | 201 | like_count = get_count('preview_like', 'like') |
202 | comment_count = get_count( | |
203 | ('preview_comment', 'to_comment', 'to_parent_comment'), 'comment') | |
204 | ||
60c8fc73 S |
205 | comments = [] |
206 | for comment in try_get(media, lambda x: x['edge_media_to_parent_comment']['edges']): | |
207 | comment_dict = comment.get('node', {}) | |
208 | comment_text = comment_dict.get('text') | |
209 | if comment_text: | |
210 | comments.append({ | |
211 | 'author': try_get(comment_dict, lambda x: x['owner']['username']), | |
212 | 'author_id': try_get(comment_dict, lambda x: x['owner']['id']), | |
213 | 'id': comment_dict.get('id'), | |
214 | 'text': comment_text, | |
215 | 'timestamp': int_or_none(comment_dict.get('created_at')), | |
216 | }) | |
29f7c58a | 217 | if not video_url: |
218 | edges = try_get( | |
219 | media, lambda x: x['edge_sidecar_to_children']['edges'], | |
220 | list) or [] | |
221 | if edges: | |
222 | entries = [] | |
223 | for edge_num, edge in enumerate(edges, start=1): | |
224 | node = try_get(edge, lambda x: x['node'], dict) | |
225 | if not node: | |
226 | continue | |
227 | node_video_url = url_or_none(node.get('video_url')) | |
228 | if not node_video_url: | |
229 | continue | |
230 | entries.append({ | |
231 | 'id': node.get('shortcode') or node['id'], | |
cce889b9 | 232 | 'title': node.get('title') or 'Video %d' % edge_num, |
29f7c58a | 233 | 'url': node_video_url, |
234 | 'thumbnail': node.get('display_url'), | |
cce889b9 | 235 | 'duration': float_or_none(node.get('video_duration')), |
29f7c58a | 236 | 'width': int_or_none(try_get(node, lambda x: x['dimensions']['width'])), |
237 | 'height': int_or_none(try_get(node, lambda x: x['dimensions']['height'])), | |
238 | 'view_count': int_or_none(node.get('video_view_count')), | |
239 | }) | |
240 | return self.playlist_result( | |
241 | entries, video_id, | |
242 | 'Post by %s' % uploader_id if uploader_id else None, | |
243 | description) | |
98960c91 S |
244 | |
245 | if not video_url: | |
246 | video_url = self._og_search_video_url(webpage, secure=False) | |
247 | ||
16097822 DR |
248 | formats = [{ |
249 | 'url': video_url, | |
250 | 'width': width, | |
251 | 'height': height, | |
252 | }] | |
253 | ||
98960c91 S |
254 | if not uploader_id: |
255 | uploader_id = self._search_regex( | |
256 | r'"owner"\s*:\s*{\s*"username"\s*:\s*"(.+?)"', | |
257 | webpage, 'uploader id', fatal=False) | |
258 | ||
259 | if not description: | |
260 | description = self._search_regex( | |
261 | r'"caption"\s*:\s*"(.+?)"', webpage, 'description', default=None) | |
262 | if description is not None: | |
263 | description = lowercase_escape(description) | |
264 | ||
265 | if not thumbnail: | |
266 | thumbnail = self._og_search_thumbnail(webpage) | |
59fc531f | 267 | |
0de668af JMF |
268 | return { |
269 | 'id': video_id, | |
16097822 | 270 | 'formats': formats, |
0de668af | 271 | 'ext': 'mp4', |
cce889b9 | 272 | 'title': title or 'Video by %s' % uploader_id, |
98960c91 | 273 | 'description': description, |
cce889b9 | 274 | 'duration': duration, |
98960c91 S |
275 | 'thumbnail': thumbnail, |
276 | 'timestamp': timestamp, | |
0de668af | 277 | 'uploader_id': uploader_id, |
98960c91 S |
278 | 'uploader': uploader, |
279 | 'like_count': like_count, | |
280 | 'comment_count': comment_count, | |
a56e74e2 | 281 | 'comments': comments, |
3dd39c5f S |
282 | 'http_headers': { |
283 | 'Referer': 'https://www.instagram.com/', | |
284 | } | |
0de668af | 285 | } |
ea38e55f PH |
286 | |
287 | ||
31fbedc0 | 288 | class InstagramPlaylistIE(InfoExtractor): |
289 | # A superclass for handling any kind of query based on GraphQL which | |
290 | # results in a playlist. | |
291 | ||
292 | _gis_tmpl = None # used to cache GIS request type | |
ea38e55f | 293 | |
31fbedc0 | 294 | def _parse_graphql(self, webpage, item_id): |
295 | # Reads a webpage and returns its GraphQL data. | |
296 | return self._parse_json( | |
297 | self._search_regex( | |
298 | r'sharedData\s*=\s*({.+?})\s*;\s*[<\n]', webpage, 'data'), | |
299 | item_id) | |
238d42cf | 300 | |
31fbedc0 | 301 | def _extract_graphql(self, data, url): |
302 | # Parses GraphQL queries containing videos and generates a playlist. | |
27b1c73f | 303 | def get_count(suffix): |
5fc12b95 | 304 | return int_or_none(try_get( |
27b1c73f RA |
305 | node, lambda x: x['edge_media_' + suffix]['count'])) |
306 | ||
31fbedc0 | 307 | uploader_id = self._match_id(url) |
dd9aea8c S |
308 | csrf_token = data['config']['csrf_token'] |
309 | rhx_gis = data.get('rhx_gis') or '3c7ca9dcefcf966d11dacf1f151335e8' | |
310 | ||
cba5d1b6 S |
311 | cursor = '' |
312 | for page_num in itertools.count(1): | |
31fbedc0 | 313 | variables = { |
9b3036bd | 314 | 'first': 12, |
dd9aea8c | 315 | 'after': cursor, |
31fbedc0 | 316 | } |
317 | variables.update(self._query_vars_for(data)) | |
318 | variables = json.dumps(variables) | |
238d42cf S |
319 | |
320 | if self._gis_tmpl: | |
321 | gis_tmpls = [self._gis_tmpl] | |
322 | else: | |
323 | gis_tmpls = [ | |
324 | '%s' % rhx_gis, | |
325 | '', | |
326 | '%s:%s' % (rhx_gis, csrf_token), | |
327 | '%s:%s:%s' % (rhx_gis, csrf_token, std_headers['User-Agent']), | |
328 | ] | |
329 | ||
31fbedc0 | 330 | # try all of the ways to generate a GIS query, and not only use the |
331 | # first one that works, but cache it for future requests | |
238d42cf S |
332 | for gis_tmpl in gis_tmpls: |
333 | try: | |
31fbedc0 | 334 | json_data = self._download_json( |
238d42cf S |
335 | 'https://www.instagram.com/graphql/query/', uploader_id, |
336 | 'Downloading JSON page %d' % page_num, headers={ | |
337 | 'X-Requested-With': 'XMLHttpRequest', | |
338 | 'X-Instagram-GIS': hashlib.md5( | |
339 | ('%s:%s' % (gis_tmpl, variables)).encode('utf-8')).hexdigest(), | |
340 | }, query={ | |
31fbedc0 | 341 | 'query_hash': self._QUERY_HASH, |
238d42cf | 342 | 'variables': variables, |
31fbedc0 | 343 | }) |
344 | media = self._parse_timeline_from(json_data) | |
238d42cf S |
345 | self._gis_tmpl = gis_tmpl |
346 | break | |
347 | except ExtractorError as e: | |
31fbedc0 | 348 | # if it's an error caused by a bad query, and there are |
349 | # more GIS templates to try, ignore it and keep trying | |
238d42cf S |
350 | if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403: |
351 | if gis_tmpl != gis_tmpls[-1]: | |
352 | continue | |
353 | raise | |
cba5d1b6 S |
354 | |
355 | edges = media.get('edges') | |
356 | if not edges or not isinstance(edges, list): | |
357 | break | |
358 | ||
359 | for edge in edges: | |
360 | node = edge.get('node') | |
361 | if not node or not isinstance(node, dict): | |
362 | continue | |
363 | if node.get('__typename') != 'GraphVideo' and node.get('is_video') is not True: | |
364 | continue | |
365 | video_id = node.get('shortcode') | |
366 | if not video_id: | |
367 | continue | |
368 | ||
369 | info = self.url_result( | |
370 | 'https://instagram.com/p/%s/' % video_id, | |
371 | ie=InstagramIE.ie_key(), video_id=video_id) | |
372 | ||
373 | description = try_get( | |
374 | node, lambda x: x['edge_media_to_caption']['edges'][0]['node']['text'], | |
375 | compat_str) | |
376 | thumbnail = node.get('thumbnail_src') or node.get('display_src') | |
377 | timestamp = int_or_none(node.get('taken_at_timestamp')) | |
378 | ||
379 | comment_count = get_count('to_comment') | |
380 | like_count = get_count('preview_like') | |
381 | view_count = int_or_none(node.get('video_view_count')) | |
382 | ||
383 | info.update({ | |
384 | 'description': description, | |
385 | 'thumbnail': thumbnail, | |
386 | 'timestamp': timestamp, | |
387 | 'comment_count': comment_count, | |
388 | 'like_count': like_count, | |
389 | 'view_count': view_count, | |
ea38e55f | 390 | }) |
cba5d1b6 S |
391 | |
392 | yield info | |
393 | ||
394 | page_info = media.get('page_info') | |
395 | if not page_info or not isinstance(page_info, dict): | |
396 | break | |
397 | ||
398 | has_next_page = page_info.get('has_next_page') | |
399 | if not has_next_page: | |
400 | break | |
401 | ||
402 | cursor = page_info.get('end_cursor') | |
403 | if not cursor or not isinstance(cursor, compat_str): | |
404 | break | |
5fc12b95 S |
405 | |
406 | def _real_extract(self, url): | |
31fbedc0 | 407 | user_or_tag = self._match_id(url) |
408 | webpage = self._download_webpage(url, user_or_tag) | |
409 | data = self._parse_graphql(webpage, user_or_tag) | |
dd9aea8c | 410 | |
31fbedc0 | 411 | self._set_cookie('instagram.com', 'ig_pr', '1') |
dd9aea8c | 412 | |
5fc12b95 | 413 | return self.playlist_result( |
31fbedc0 | 414 | self._extract_graphql(data, url), user_or_tag, user_or_tag) |
415 | ||
416 | ||
417 | class InstagramUserIE(InstagramPlaylistIE): | |
418 | _VALID_URL = r'https?://(?:www\.)?instagram\.com/(?P<id>[^/]{2,})/?(?:$|[?#])' | |
419 | IE_DESC = 'Instagram user profile' | |
420 | IE_NAME = 'instagram:user' | |
421 | _TEST = { | |
422 | 'url': 'https://instagram.com/porsche', | |
423 | 'info_dict': { | |
424 | 'id': 'porsche', | |
425 | 'title': 'porsche', | |
426 | }, | |
427 | 'playlist_count': 5, | |
428 | 'params': { | |
429 | 'extract_flat': True, | |
430 | 'skip_download': True, | |
431 | 'playlistend': 5, | |
432 | } | |
433 | } | |
434 | ||
435 | _QUERY_HASH = '42323d64886122307be10013ad2dcc44', | |
436 | ||
437 | @staticmethod | |
438 | def _parse_timeline_from(data): | |
439 | # extracts the media timeline data from a GraphQL result | |
440 | return data['data']['user']['edge_owner_to_timeline_media'] | |
441 | ||
442 | @staticmethod | |
443 | def _query_vars_for(data): | |
444 | # returns a dictionary of variables to add to the timeline query based | |
445 | # on the GraphQL of the original page | |
446 | return { | |
447 | 'id': data['entry_data']['ProfilePage'][0]['graphql']['user']['id'] | |
448 | } | |
449 | ||
450 | ||
451 | class InstagramTagIE(InstagramPlaylistIE): | |
452 | _VALID_URL = r'https?://(?:www\.)?instagram\.com/explore/tags/(?P<id>[^/]+)' | |
453 | IE_DESC = 'Instagram hashtag search' | |
454 | IE_NAME = 'instagram:tag' | |
455 | _TEST = { | |
456 | 'url': 'https://instagram.com/explore/tags/lolcats', | |
457 | 'info_dict': { | |
458 | 'id': 'lolcats', | |
459 | 'title': 'lolcats', | |
460 | }, | |
461 | 'playlist_count': 50, | |
462 | 'params': { | |
463 | 'extract_flat': True, | |
464 | 'skip_download': True, | |
465 | 'playlistend': 50, | |
466 | } | |
467 | } | |
468 | ||
469 | _QUERY_HASH = 'f92f56d47dc7a55b606908374b43a314', | |
470 | ||
471 | @staticmethod | |
472 | def _parse_timeline_from(data): | |
473 | # extracts the media timeline data from a GraphQL result | |
474 | return data['data']['hashtag']['edge_hashtag_to_media'] | |
475 | ||
476 | @staticmethod | |
477 | def _query_vars_for(data): | |
478 | # returns a dictionary of variables to add to the timeline query based | |
479 | # on the GraphQL of the original page | |
480 | return { | |
481 | 'tag_name': | |
482 | data['entry_data']['TagPage'][0]['graphql']['hashtag']['name'] | |
483 | } |