]>
Commit | Line | Data |
---|---|---|
dcdb292f | 1 | # coding: utf-8 |
c060b774 PH |
2 | from __future__ import unicode_literals |
3 | ||
ae287755 PH |
4 | |
5 | from .common import InfoExtractor | |
c678192a ZF |
6 | from ..utils import ( |
7 | ExtractorError, | |
8 | int_or_none, | |
bed30106 | 9 | traverse_obj, |
c678192a ZF |
10 | urlencode_postdata |
11 | ) | |
ae287755 PH |
12 | |
13 | ||
14 | class TumblrIE(InfoExtractor): | |
afca767d | 15 | _VALID_URL = r'https?://(?P<blog_name>[^/?#&]+)\.tumblr\.com/(?:post|video)/(?P<id>[0-9]+)(?:$|[/?#])' |
c678192a ZF |
16 | _NETRC_MACHINE = 'tumblr' |
17 | _LOGIN_URL = 'https://www.tumblr.com/login' | |
bed30106 | 18 | _OAUTH_URL = 'https://www.tumblr.com/api/v2/oauth2/token' |
62f1f950 | 19 | _TESTS = [{ |
c060b774 | 20 | 'url': 'http://tatianamaslanydaily.tumblr.com/post/54196191430/orphan-black-dvd-extra-behind-the-scenes', |
c060b774 PH |
21 | 'md5': '479bb068e5b16462f5176a6828829767', |
22 | 'info_dict': { | |
62f1f950 PP |
23 | 'id': '54196191430', |
24 | 'ext': 'mp4', | |
bed30106 | 25 | 'title': 'md5:dfac39636969fe6bf1caa2d50405f069', |
403be2ee | 26 | 'description': 'md5:390ab77358960235b6937ab3b8528956', |
bed30106 | 27 | 'uploader_id': 'tatianamaslanydaily', |
28 | 'uploader_url': 'https://tatianamaslanydaily.tumblr.com/', | |
403be2ee | 29 | 'thumbnail': r're:^https?://.*\.jpg', |
30 | 'duration': 127, | |
bed30106 | 31 | 'like_count': int, |
32 | 'repost_count': int, | |
33 | 'age_limit': 0, | |
34 | 'tags': ['Orphan Black', 'Tatiana Maslany', 'Interview', 'Video', 'OB S1 DVD Extras'], | |
6f5ac90c | 35 | } |
62f1f950 | 36 | }, { |
bed30106 | 37 | 'note': 'multiple formats', |
403be2ee | 38 | 'url': 'https://maskofthedragon.tumblr.com/post/626907179849564160/mona-talking-in-english', |
39 | 'md5': 'f43ff8a8861712b6cf0e0c2bd84cfc68', | |
62f1f950 | 40 | 'info_dict': { |
403be2ee | 41 | 'id': '626907179849564160', |
62f1f950 | 42 | 'ext': 'mp4', |
bed30106 | 43 | 'title': 'Mona\xa0“talking” in\xa0“english”', |
403be2ee | 44 | 'description': 'md5:082a3a621530cb786ad2b7592a6d9e2c', |
bed30106 | 45 | 'uploader_id': 'maskofthedragon', |
46 | 'uploader_url': 'https://maskofthedragon.tumblr.com/', | |
403be2ee | 47 | 'thumbnail': r're:^https?://.*\.jpg', |
48 | 'duration': 7, | |
bed30106 | 49 | 'like_count': int, |
50 | 'repost_count': int, | |
51 | 'age_limit': 0, | |
52 | 'tags': 'count:19', | |
2a27e662 OA |
53 | }, |
54 | 'params': { | |
55 | 'format': 'hd', | |
56 | }, | |
bed30106 | 57 | }, { |
58 | 'note': 'non-iframe video (with related posts)', | |
59 | 'url': 'https://shieldfoss.tumblr.com/post/675519763813908480', | |
60 | 'md5': '12bdb75661ef443bffe5a4dac1dbf118', | |
61 | 'info_dict': { | |
62 | 'id': '675519763813908480', | |
63 | 'ext': 'mp4', | |
64 | 'title': 'Shieldfoss', | |
65 | 'uploader_id': 'nerviovago', | |
66 | 'uploader_url': 'https://nerviovago.tumblr.com/', | |
67 | 'thumbnail': r're:^https?://.*\.jpg', | |
68 | 'like_count': int, | |
69 | 'repost_count': int, | |
70 | 'age_limit': 0, | |
71 | 'tags': [], | |
72 | } | |
73 | }, { | |
74 | 'note': 'dashboard only (original post)', | |
75 | 'url': 'https://jujanon.tumblr.com/post/159704441298/my-baby-eating', | |
76 | 'md5': '029f7c91ab386701b211e3d494d2d95e', | |
77 | 'info_dict': { | |
78 | 'id': '159704441298', | |
79 | 'ext': 'mp4', | |
80 | 'title': 'md5:ba79365861101f4911452728d2950561', | |
81 | 'description': 'md5:773738196cea76b6996ec71e285bdabc', | |
82 | 'uploader_id': 'jujanon', | |
83 | 'uploader_url': 'https://jujanon.tumblr.com/', | |
84 | 'thumbnail': r're:^https?://.*\.jpg', | |
85 | 'like_count': int, | |
86 | 'repost_count': int, | |
87 | 'age_limit': 0, | |
88 | 'tags': ['crabs', 'my video', 'my pets'], | |
89 | } | |
90 | }, { | |
91 | 'note': 'dashboard only (reblog)', | |
92 | 'url': 'https://bartlebyshop.tumblr.com/post/180294460076/duality-of-bird', | |
93 | 'md5': '04334e7cadb1af680d162912559f51a5', | |
94 | 'info_dict': { | |
95 | 'id': '180294460076', | |
96 | 'ext': 'mp4', | |
97 | 'title': 'duality of bird', | |
98 | 'description': 'duality of bird', | |
99 | 'uploader_id': 'todaysbird', | |
100 | 'uploader_url': 'https://todaysbird.tumblr.com/', | |
101 | 'thumbnail': r're:^https?://.*\.jpg', | |
102 | 'like_count': int, | |
103 | 'repost_count': int, | |
104 | 'age_limit': 0, | |
105 | 'tags': [], | |
106 | } | |
107 | }, { | |
108 | 'note': 'dashboard only (external)', | |
109 | 'url': 'https://afloweroutofstone.tumblr.com/post/675661759168823296/the-blues-remembers-everything-the-country-forgot', | |
110 | 'info_dict': { | |
111 | 'id': 'q67_fd7b8SU', | |
112 | 'ext': 'mp4', | |
113 | 'title': 'The Blues Remembers Everything the Country Forgot', | |
114 | 'alt_title': 'The Blues Remembers Everything the Country Forgot', | |
115 | 'description': 'md5:1a6b4097e451216835a24c1023707c79', | |
116 | 'release_date': '20201224', | |
117 | 'creator': 'md5:c2239ba15430e87c3b971ba450773272', | |
118 | 'uploader': 'Moor Mother - Topic', | |
119 | 'upload_date': '20201223', | |
120 | 'uploader_id': 'UCxrMtFBRkFvQJ_vVM4il08w', | |
121 | 'uploader_url': 'http://www.youtube.com/channel/UCxrMtFBRkFvQJ_vVM4il08w', | |
122 | 'thumbnail': r're:^https?://i.ytimg.com/.*', | |
123 | 'channel': 'Moor Mother - Topic', | |
124 | 'channel_id': 'UCxrMtFBRkFvQJ_vVM4il08w', | |
125 | 'channel_url': 'https://www.youtube.com/channel/UCxrMtFBRkFvQJ_vVM4il08w', | |
126 | 'channel_follower_count': int, | |
127 | 'duration': 181, | |
128 | 'view_count': int, | |
129 | 'like_count': int, | |
130 | 'age_limit': 0, | |
131 | 'categories': ['Music'], | |
132 | 'tags': 'count:7', | |
133 | 'live_status': 'not_live', | |
134 | 'playable_in_embed': True, | |
135 | 'availability': 'public', | |
136 | 'track': 'The Blues Remembers Everything the Country Forgot', | |
137 | 'artist': 'md5:c2239ba15430e87c3b971ba450773272', | |
138 | 'album': 'Brass', | |
139 | 'release_year': 2020, | |
140 | }, | |
141 | 'add_ie': ['Youtube'], | |
8f947841 YCH |
142 | }, { |
143 | 'url': 'http://naked-yogi.tumblr.com/post/118312946248/naked-smoking-stretching', | |
144 | 'md5': 'de07e5211d60d4f3a2c3df757ea9f6ab', | |
145 | 'info_dict': { | |
146 | 'id': 'Wmur', | |
147 | 'ext': 'mp4', | |
148 | 'title': 'naked smoking & stretching', | |
149 | 'upload_date': '20150506', | |
150 | 'timestamp': 1430931613, | |
88c86d21 S |
151 | 'age_limit': 18, |
152 | 'uploader_id': '1638622', | |
153 | 'uploader': 'naked-yogi', | |
8f947841 | 154 | }, |
403be2ee | 155 | # 'add_ie': ['Vidme'], |
156 | 'skip': 'dead embedded video host' | |
bed30106 | 157 | }, { |
158 | 'url': 'https://prozdvoices.tumblr.com/post/673201091169681408/what-recording-voice-acting-sounds-like', | |
159 | 'md5': 'a0063fc8110e6c9afe44065b4ea68177', | |
160 | 'info_dict': { | |
161 | 'id': 'eomhW5MLGWA', | |
162 | 'ext': 'mp4', | |
163 | 'title': 'what recording voice acting sounds like', | |
164 | 'description': 'md5:1da3faa22d0e0b1d8b50216c284ee798', | |
165 | 'uploader': 'ProZD', | |
166 | 'upload_date': '20220112', | |
167 | 'uploader_id': 'ProZD', | |
168 | 'uploader_url': 'http://www.youtube.com/user/ProZD', | |
169 | 'thumbnail': r're:^https?://i.ytimg.com/.*', | |
170 | 'channel': 'ProZD', | |
171 | 'channel_id': 'UC6MFZAOHXlKK1FI7V0XQVeA', | |
172 | 'channel_url': 'https://www.youtube.com/channel/UC6MFZAOHXlKK1FI7V0XQVeA', | |
173 | 'channel_follower_count': int, | |
174 | 'duration': 20, | |
175 | 'view_count': int, | |
176 | 'like_count': int, | |
177 | 'age_limit': 0, | |
178 | 'categories': ['Film & Animation'], | |
179 | 'tags': [], | |
180 | 'live_status': 'not_live', | |
181 | 'playable_in_embed': True, | |
182 | 'availability': 'public', | |
183 | }, | |
184 | 'add_ie': ['Youtube'], | |
c5895d5d | 185 | }, { |
403be2ee | 186 | 'url': 'https://dominustempori.tumblr.com/post/673572712813297664/youtubes-all-right-for-some-pretty-cool', |
bed30106 | 187 | 'md5': '203e9eb8077e3f45bfaeb4c86c1467b8', |
c5895d5d | 188 | 'info_dict': { |
403be2ee | 189 | 'id': '87816359', |
bed30106 | 190 | 'ext': 'mov', |
403be2ee | 191 | 'title': 'Harold Ramis', |
bed30106 | 192 | 'description': 'md5:be8e68cbf56ce0785c77f0c6c6dfaf2c', |
403be2ee | 193 | 'uploader': 'Resolution Productions Group', |
194 | 'uploader_id': 'resolutionproductions', | |
195 | 'uploader_url': 'https://vimeo.com/resolutionproductions', | |
bed30106 | 196 | 'upload_date': '20140227', |
403be2ee | 197 | 'thumbnail': r're:^https?://i.vimeocdn.com/video/.*', |
bed30106 | 198 | 'timestamp': 1393523719, |
403be2ee | 199 | 'duration': 291, |
c5895d5d YCH |
200 | }, |
201 | 'add_ie': ['Vimeo'], | |
fc27ea94 YCH |
202 | }, { |
203 | 'url': 'http://sutiblr.tumblr.com/post/139638707273', | |
204 | 'md5': '2dd184b3669e049ba40563a7d423f95c', | |
205 | 'info_dict': { | |
206 | 'id': 'ir7qBEIKqvq', | |
207 | 'ext': 'mp4', | |
208 | 'title': 'Vine by sutiblr', | |
209 | 'alt_title': 'Vine by sutiblr', | |
210 | 'uploader': 'sutiblr', | |
211 | 'uploader_id': '1198993975374495744', | |
212 | 'upload_date': '20160220', | |
213 | 'like_count': int, | |
214 | 'comment_count': int, | |
215 | 'repost_count': int, | |
403be2ee | 216 | 'thumbnail': r're:^https?://.*\.jpg', |
217 | 'timestamp': 1455940159, | |
218 | 'view_count': int, | |
fc27ea94 YCH |
219 | }, |
220 | 'add_ie': ['Vine'], | |
32d88410 | 221 | }, { |
403be2ee | 222 | 'url': 'https://silami.tumblr.com/post/84250043974/my-bad-river-flows-in-you-impression-on-maschine', |
223 | 'md5': '3c92d7c3d867f14ccbeefa2119022277', | |
32d88410 | 224 | 'info_dict': { |
403be2ee | 225 | 'id': 'nYtvtTPuTl', |
32d88410 | 226 | 'ext': 'mp4', |
403be2ee | 227 | 'title': 'Video by silbulterman', |
228 | 'description': '#maschine', | |
229 | 'uploader_id': '242859024', | |
230 | 'thumbnail': r're:^https?://.*\.jpg', | |
231 | 'timestamp': 1398801174, | |
232 | 'like_count': int, | |
233 | 'uploader': 'Sil', | |
234 | 'channel': 'silbulterman', | |
235 | 'comment_count': int, | |
236 | 'upload_date': '20140429', | |
32d88410 YCH |
237 | }, |
238 | 'add_ie': ['Instagram'], | |
62f1f950 | 239 | }] |
ae287755 | 240 | |
bed30106 | 241 | _providers = { |
242 | 'instagram': 'Instagram', | |
243 | 'vimeo': 'Vimeo', | |
244 | 'vine': 'Vine', | |
245 | 'youtube': 'Youtube', | |
246 | } | |
247 | ||
248 | _ACCESS_TOKEN = None | |
249 | ||
c678192a | 250 | def _real_initialize(self): |
bed30106 | 251 | self.get_access_token() |
c678192a ZF |
252 | self._login() |
253 | ||
bed30106 | 254 | def get_access_token(self): |
255 | login_page = self._download_webpage( | |
256 | self._LOGIN_URL, None, 'Downloading login page', fatal=False) | |
257 | if login_page: | |
258 | self._ACCESS_TOKEN = self._search_regex( | |
259 | r'"API_TOKEN":\s*"(\w+)"', login_page, 'API access token', fatal=False) | |
260 | if not self._ACCESS_TOKEN: | |
261 | self.report_warning('Failed to get access token; metadata will be missing and some videos may not work') | |
262 | ||
c678192a | 263 | def _login(self): |
68217024 | 264 | username, password = self._get_login_info() |
bed30106 | 265 | if not username: |
c678192a | 266 | return |
56cd31f3 | 267 | |
bed30106 | 268 | if not self._ACCESS_TOKEN: |
56cd31f3 S |
269 | return |
270 | ||
bed30106 | 271 | self._download_json( |
272 | self._OAUTH_URL, None, 'Logging in', | |
273 | data=urlencode_postdata({ | |
274 | 'password': password, | |
275 | 'grant_type': 'password', | |
276 | 'username': username, | |
277 | }), headers={ | |
278 | 'Content-Type': 'application/x-www-form-urlencoded', | |
279 | 'Authorization': f'Bearer {self._ACCESS_TOKEN}', | |
280 | }, | |
281 | errnote='Login failed', fatal=False) | |
c678192a | 282 | |
ae287755 | 283 | def _real_extract(self, url): |
bed30106 | 284 | blog, video_id = self._match_valid_url(url).groups() |
ae287755 | 285 | |
bed30106 | 286 | url = f'http://{blog}.tumblr.com/post/{video_id}/' |
c909e582 | 287 | webpage, urlh = self._download_webpage_handle(url, video_id) |
c5895d5d | 288 | |
7947a1f7 | 289 | redirect_url = urlh.geturl() |
97b01144 | 290 | |
bed30106 | 291 | api_only = bool(self._search_regex( |
292 | r'(tumblr.com|^)/(safe-mode|login_required|blog/view)', | |
293 | redirect_url, 'redirect', default=None)) | |
294 | ||
295 | if api_only and not self._ACCESS_TOKEN: | |
296 | raise ExtractorError('Cannot get data for dashboard-only post without access token') | |
297 | ||
298 | post_json = {} | |
299 | if self._ACCESS_TOKEN: | |
300 | post_json = traverse_obj( | |
301 | self._download_json( | |
302 | f'https://www.tumblr.com/api/v2/blog/{blog}/posts/{video_id}/permalink', | |
303 | video_id, headers={'Authorization': f'Bearer {self._ACCESS_TOKEN}'}, fatal=False), | |
304 | ('response', 'timeline', 'elements', 0)) or {} | |
305 | content_json = traverse_obj(post_json, ('trail', 0, 'content'), ('content')) or [] | |
306 | video_json = next( | |
307 | (item for item in content_json if item.get('type') == 'video'), {}) | |
308 | media_json = video_json.get('media') or {} | |
309 | if api_only and not media_json.get('url') and not video_json.get('url'): | |
310 | raise ExtractorError('Failed to find video data for dashboard-only post') | |
311 | ||
312 | if not media_json.get('url') and video_json.get('url'): | |
313 | # external video host | |
314 | return self.url_result( | |
315 | video_json['url'], | |
316 | self._providers.get(video_json.get('provider'), 'Generic')) | |
317 | ||
318 | video_url = self._og_search_video_url(webpage, default=None) | |
319 | duration = None | |
320 | formats = [] | |
321 | ||
322 | # iframes can supply duration and sometimes additional formats, so check for one | |
681b9caa | 323 | iframe_url = self._search_regex( |
bed30106 | 324 | fr'src=\'(https?://www\.tumblr\.com/video/{blog}/{video_id}/[^\']+)\'', |
c909e582 | 325 | webpage, 'iframe url', default=None) |
bed30106 | 326 | if iframe_url: |
327 | iframe = self._download_webpage( | |
328 | iframe_url, video_id, 'Downloading iframe page', | |
329 | headers={'Referer': redirect_url}) | |
330 | ||
331 | options = self._parse_json( | |
332 | self._search_regex( | |
333 | r'data-crt-options=(["\'])(?P<options>.+?)\1', iframe, | |
334 | 'hd video url', default='', group='options'), | |
335 | video_id, fatal=False) | |
336 | if options: | |
337 | duration = int_or_none(options.get('duration')) | |
338 | ||
339 | hd_url = options.get('hdUrl') | |
340 | if hd_url: | |
341 | # there are multiple formats; extract them | |
342 | # ignore other sources of width/height data as they may be wrong | |
343 | sources = [] | |
344 | sd_url = self._search_regex( | |
345 | r'<source[^>]+src=(["\'])(?P<url>.+?)\1', iframe, | |
346 | 'sd video url', default=None, group='url') | |
347 | if sd_url: | |
348 | sources.append((sd_url, 'sd')) | |
349 | sources.append((hd_url, 'hd')) | |
350 | ||
351 | formats = [{ | |
352 | 'url': video_url, | |
353 | 'format_id': format_id, | |
354 | 'height': int_or_none(self._search_regex( | |
355 | r'_(\d+)\.\w+$', video_url, 'height', default=None)), | |
356 | 'quality': quality, | |
357 | } for quality, (video_url, format_id) in enumerate(sources)] | |
358 | ||
359 | if not media_json.get('url') and not video_url and not iframe_url: | |
360 | # external video host (but we weren't able to figure it out from the api) | |
403be2ee | 361 | iframe_url = self._search_regex( |
362 | r'src=["\'](https?://safe\.txmblr\.com/svc/embed/inline/[^"\']+)["\']', | |
363 | webpage, 'embed iframe url', default=None) | |
364 | return self.url_result(iframe_url or redirect_url, 'Generic') | |
365 | ||
bed30106 | 366 | formats = formats or [{ |
367 | 'url': media_json.get('url') or video_url, | |
368 | 'width': int_or_none( | |
369 | media_json.get('width') or self._og_search_property('video:width', webpage, default=None)), | |
370 | 'height': int_or_none( | |
371 | media_json.get('height') or self._og_search_property('video:height', webpage, default=None)), | |
372 | }] | |
140ac739 | 373 | self._sort_formats(formats) |
ae287755 | 374 | |
bed30106 | 375 | # the url we're extracting from might be an original post or it might be a reblog. |
376 | # if it's a reblog, og:description will be the reblogger's comment, not the uploader's. | |
377 | # content_json is always the op, so if it exists but has no text, there's no description | |
378 | if content_json: | |
379 | description = '\n\n'.join(( | |
380 | item.get('text') for item in content_json if item.get('type') == 'text')) or None | |
381 | else: | |
382 | description = self._og_search_description(webpage, default=None) | |
383 | uploader_id = traverse_obj(post_json, 'reblogged_root_name', 'blog_name') | |
ae287755 | 384 | |
3da0e1f8 PH |
385 | return { |
386 | 'id': video_id, | |
bed30106 | 387 | 'title': post_json.get('summary') or (blog if api_only else self._html_search_regex( |
388 | r'(?s)<title>(?P<title>.*?)(?: \| Tumblr)?</title>', webpage, 'title')), | |
389 | 'description': description, | |
390 | 'thumbnail': (traverse_obj(video_json, ('poster', 0, 'url')) | |
391 | or self._og_search_thumbnail(webpage, default=None)), | |
392 | 'uploader_id': uploader_id, | |
393 | 'uploader_url': f'https://{uploader_id}.tumblr.com/' if uploader_id else None, | |
140ac739 | 394 | 'duration': duration, |
bed30106 | 395 | 'like_count': post_json.get('like_count'), |
396 | 'repost_count': post_json.get('reblog_count'), | |
397 | 'age_limit': {True: 18, False: 0}.get(post_json.get('is_nsfw')), | |
398 | 'tags': post_json.get('tags'), | |
140ac739 | 399 | 'formats': formats, |
3da0e1f8 | 400 | } |