]>
jfr.im git - yt-dlp.git/blob - youtube_dlc/extractor/instagram.py
c3eba011442353a4e3c377ee68737993b2573a38
1 from __future__
import unicode_literals
8 from .common
import InfoExtractor
15 get_element_by_attribute
,
24 class InstagramIE(InfoExtractor
):
25 _VALID_URL
= r
'(?P<url>https?://(?:www\.)?instagram\.com/(?:p|tv)/(?P<id>[^/?#&]+))'
27 'url': 'https://instagram.com/p/aye83DjauH/?foo=bar#abc',
28 'md5': '0d2da106a9d2631273e192b372806516',
32 'title': 'Video by naomipq',
33 'description': 'md5:1f17f0ab29bd6fe2bfad705f58de3cb8',
34 'thumbnail': r
're:^https?://.*\.jpg',
35 'timestamp': 1371748545,
36 'upload_date': '20130620',
37 'uploader_id': 'naomipq',
38 'uploader': 'Naomi Leonor Phan-Quang',
45 'url': 'https://www.instagram.com/p/BA-pQFBG8HZ/?taken-by=britneyspears',
49 'title': 'Video by britneyspears',
50 'thumbnail': r
're:^https?://.*\.jpg',
51 'timestamp': 1453760977,
52 'upload_date': '20160125',
53 'uploader_id': 'britneyspears',
54 'uploader': 'Britney Spears',
60 'skip_download': True,
64 'url': 'https://www.instagram.com/p/BQ0eAlwhDrw/',
86 'title': 'Post by instagram',
87 'description': 'md5:0f9203fc6a2ce4d228da5754bcf54957',
90 'url': 'https://instagram.com/p/-Cmh1cukG2/',
91 'only_matching': True,
93 'url': 'http://instagram.com/p/9o6LshA7zy/embed/',
94 'only_matching': True,
96 'url': 'https://www.instagram.com/tv/aye83DjauH/',
97 'only_matching': True,
101 def _extract_embed_url(webpage
):
103 r
'<iframe[^>]+src=(["\'])(?P
<url
>(?
:https?
:)?
//(?
:www\
.)?instagram\
.com
/p
/[^
/]+/embed
.*?
)\
1',
106 return mobj.group('url
')
108 blockquote_el = get_element_by_attribute(
109 'class', 'instagram
-media
', webpage)
110 if blockquote_el is None:
114 r'<a
[^
>]+href
=([\'"])(?P<link>[^\'"]+)\
1', blockquote_el)
116 return mobj.group('link
')
118 def _real_extract(self, url):
119 mobj = re.match(self._VALID_URL, url)
120 video_id = mobj.group('id')
121 url = mobj.group('url
')
123 webpage = self._download_webpage(url, video_id)
125 (video_url, description, thumbnail, timestamp, uploader,
126 uploader_id, like_count, comment_count, comments, height,
129 shared_data = try_get(webpage,
130 (lambda x: self._parse_json(
132 r'window\
.__additionalDataLoaded\
(\'/(?
:p|tv
)/(?
:[^
/?
#&]+)/\',({.+?})\);',
133 x
, 'additional data', default
='{}'),
134 video_id
, fatal
=False),
135 lambda x
: self
._parse
_json
(
137 r
'window\._sharedData\s*=\s*({.+?});',
138 x
, 'shared data', default
='{}'),
139 video_id
, fatal
=False)['entry_data']['PostPage'][0]),
144 (lambda x
: x
['graphql']['shortcode_media'],
145 lambda x
: x
['media']),
148 video_url
= media
.get('video_url')
149 height
= int_or_none(media
.get('dimensions', {}).get('height'))
150 width
= int_or_none(media
.get('dimensions', {}).get('width'))
151 description
= try_get(
152 media
, lambda x
: x
['edge_media_to_caption']['edges'][0]['node']['text'],
153 compat_str
) or media
.get('caption')
154 thumbnail
= media
.get('display_src') or media
.get('thumbnail_src')
155 timestamp
= int_or_none(media
.get('taken_at_timestamp') or media
.get('date'))
156 uploader
= media
.get('owner', {}).get('full_name')
157 uploader_id
= media
.get('owner', {}).get('username')
159 def get_count(key
, kind
):
160 return int_or_none(try_get(
161 media
, (lambda x
: x
['edge_media_%s' % key
]['count'],
162 lambda x
: x
['%ss' % kind
]['count'])))
163 like_count
= get_count('preview_like', 'like')
164 comment_count
= get_count('to_comment', 'comment')
167 'author': comment
.get('user', {}).get('username'),
168 'author_id': comment
.get('user', {}).get('id'),
169 'id': comment
.get('id'),
170 'text': comment
.get('text'),
171 'timestamp': int_or_none(comment
.get('created_at')),
172 } for comment
in media
.get(
173 'comments', {}).get('nodes', []) if comment
.get('text')]
176 media
, lambda x
: x
['edge_sidecar_to_children']['edges'],
180 for edge_num
, edge
in enumerate(edges
, start
=1):
181 node
= try_get(edge
, lambda x
: x
['node'], dict)
184 node_video_url
= url_or_none(node
.get('video_url'))
185 if not node_video_url
:
188 'id': node
.get('shortcode') or node
['id'],
189 'title': 'Video %d' % edge_num
,
190 'url': node_video_url
,
191 'thumbnail': node
.get('display_url'),
192 'width': int_or_none(try_get(node
, lambda x
: x
['dimensions']['width'])),
193 'height': int_or_none(try_get(node
, lambda x
: x
['dimensions']['height'])),
194 'view_count': int_or_none(node
.get('video_view_count')),
196 return self
.playlist_result(
198 'Post by %s' % uploader_id
if uploader_id
else None,
202 video_url
= self
._og
_search
_video
_url
(webpage
, secure
=False)
211 uploader_id
= self
._search
_regex
(
212 r
'"owner"\s*:\s*{\s*"username"\s*:\s*"(.+?)"',
213 webpage
, 'uploader id', fatal
=False)
216 description
= self
._search
_regex
(
217 r
'"caption"\s*:\s*"(.+?)"', webpage
, 'description', default
=None)
218 if description
is not None:
219 description
= lowercase_escape(description
)
222 thumbnail
= self
._og
_search
_thumbnail
(webpage
)
228 'title': 'Video by %s' % uploader_id
,
229 'description': description
,
230 'thumbnail': thumbnail
,
231 'timestamp': timestamp
,
232 'uploader_id': uploader_id
,
233 'uploader': uploader
,
234 'like_count': like_count
,
235 'comment_count': comment_count
,
236 'comments': comments
,
240 class InstagramPlaylistIE(InfoExtractor
):
241 # A superclass for handling any kind of query based on GraphQL which
242 # results in a playlist.
244 _gis_tmpl
= None # used to cache GIS request type
246 def _parse_graphql(self
, webpage
, item_id
):
247 # Reads a webpage and returns its GraphQL data.
248 return self
._parse
_json
(
250 r
'sharedData\s*=\s*({.+?})\s*;\s*[<\n]', webpage
, 'data'),
253 def _extract_graphql(self
, data
, url
):
254 # Parses GraphQL queries containing videos and generates a playlist.
255 def get_count(suffix
):
256 return int_or_none(try_get(
257 node
, lambda x
: x
['edge_media_' + suffix
]['count']))
259 uploader_id
= self
._match
_id
(url
)
260 csrf_token
= data
['config']['csrf_token']
261 rhx_gis
= data
.get('rhx_gis') or '3c7ca9dcefcf966d11dacf1f151335e8'
264 for page_num
in itertools
.count(1):
269 variables
.update(self
._query
_vars
_for
(data
))
270 variables
= json
.dumps(variables
)
273 gis_tmpls
= [self
._gis
_tmpl
]
278 '%s:%s' % (rhx_gis
, csrf_token
),
279 '%s:%s:%s' % (rhx_gis
, csrf_token
, std_headers
['User-Agent']),
282 # try all of the ways to generate a GIS query, and not only use the
283 # first one that works, but cache it for future requests
284 for gis_tmpl
in gis_tmpls
:
286 json_data
= self
._download
_json
(
287 'https://www.instagram.com/graphql/query/', uploader_id
,
288 'Downloading JSON page %d' % page_num
, headers
={
289 'X-Requested-With': 'XMLHttpRequest',
290 'X-Instagram-GIS': hashlib
.md5(
291 ('%s:%s' % (gis_tmpl
, variables
)).encode('utf-8')).hexdigest(),
293 'query_hash': self
._QUERY
_HASH
,
294 'variables': variables
,
296 media
= self
._parse
_timeline
_from
(json_data
)
297 self
._gis
_tmpl
= gis_tmpl
299 except ExtractorError
as e
:
300 # if it's an error caused by a bad query, and there are
301 # more GIS templates to try, ignore it and keep trying
302 if isinstance(e
.cause
, compat_HTTPError
) and e
.cause
.code
== 403:
303 if gis_tmpl
!= gis_tmpls
[-1]:
307 edges
= media
.get('edges')
308 if not edges
or not isinstance(edges
, list):
312 node
= edge
.get('node')
313 if not node
or not isinstance(node
, dict):
315 if node
.get('__typename') != 'GraphVideo' and node
.get('is_video') is not True:
317 video_id
= node
.get('shortcode')
321 info
= self
.url_result(
322 'https://instagram.com/p/%s/' % video_id
,
323 ie
=InstagramIE
.ie_key(), video_id
=video_id
)
325 description
= try_get(
326 node
, lambda x
: x
['edge_media_to_caption']['edges'][0]['node']['text'],
328 thumbnail
= node
.get('thumbnail_src') or node
.get('display_src')
329 timestamp
= int_or_none(node
.get('taken_at_timestamp'))
331 comment_count
= get_count('to_comment')
332 like_count
= get_count('preview_like')
333 view_count
= int_or_none(node
.get('video_view_count'))
336 'description': description
,
337 'thumbnail': thumbnail
,
338 'timestamp': timestamp
,
339 'comment_count': comment_count
,
340 'like_count': like_count
,
341 'view_count': view_count
,
346 page_info
= media
.get('page_info')
347 if not page_info
or not isinstance(page_info
, dict):
350 has_next_page
= page_info
.get('has_next_page')
351 if not has_next_page
:
354 cursor
= page_info
.get('end_cursor')
355 if not cursor
or not isinstance(cursor
, compat_str
):
358 def _real_extract(self
, url
):
359 user_or_tag
= self
._match
_id
(url
)
360 webpage
= self
._download
_webpage
(url
, user_or_tag
)
361 data
= self
._parse
_graphql
(webpage
, user_or_tag
)
363 self
._set
_cookie
('instagram.com', 'ig_pr', '1')
365 return self
.playlist_result(
366 self
._extract
_graphql
(data
, url
), user_or_tag
, user_or_tag
)
369 class InstagramUserIE(InstagramPlaylistIE
):
370 _VALID_URL
= r
'https?://(?:www\.)?instagram\.com/(?P<id>[^/]{2,})/?(?:$|[?#])'
371 IE_DESC
= 'Instagram user profile'
372 IE_NAME
= 'instagram:user'
374 'url': 'https://instagram.com/porsche',
381 'extract_flat': True,
382 'skip_download': True,
387 _QUERY_HASH
= '42323d64886122307be10013ad2dcc44',
390 def _parse_timeline_from(data
):
391 # extracts the media timeline data from a GraphQL result
392 return data
['data']['user']['edge_owner_to_timeline_media']
395 def _query_vars_for(data
):
396 # returns a dictionary of variables to add to the timeline query based
397 # on the GraphQL of the original page
399 'id': data
['entry_data']['ProfilePage'][0]['graphql']['user']['id']
403 class InstagramTagIE(InstagramPlaylistIE
):
404 _VALID_URL
= r
'https?://(?:www\.)?instagram\.com/explore/tags/(?P<id>[^/]+)'
405 IE_DESC
= 'Instagram hashtag search'
406 IE_NAME
= 'instagram:tag'
408 'url': 'https://instagram.com/explore/tags/lolcats',
413 'playlist_count': 50,
415 'extract_flat': True,
416 'skip_download': True,
421 _QUERY_HASH
= 'f92f56d47dc7a55b606908374b43a314',
424 def _parse_timeline_from(data
):
425 # extracts the media timeline data from a GraphQL result
426 return data
['data']['hashtag']['edge_hashtag_to_media']
429 def _query_vars_for(data
):
430 # returns a dictionary of variables to add to the timeline query based
431 # on the GraphQL of the original page
434 data
['entry_data']['TagPage'][0]['graphql']['hashtag']['name']