2 from __future__
import unicode_literals
7 from .common
import InfoExtractor
9 compat_etree_fromstring
,
12 compat_urllib_parse_unquote
,
13 compat_urllib_parse_unquote_plus
,
30 class FacebookIE(InfoExtractor
):
34 (?:[\w-]+\.)?(?:facebook\.com|facebookcorewwwi\.onion)/
43 )\?(?:.*?)(?:v|video_id|story_fbid)=|
44 [^/]+/videos/(?:[^/]+/)?|
46 groups/[^/]+/permalink/
52 _LOGIN_URL
= 'https://www.facebook.com/login.php?next=http%3A%2F%2Ffacebook.com%2Fhome.php&login_attempt=1'
53 _CHECKPOINT_URL
= 'https://www.facebook.com/checkpoint/?next=http%3A%2F%2Ffacebook.com%2Fhome.php&_fb_noscript=1'
54 _NETRC_MACHINE
= 'facebook'
57 _CHROME_USER_AGENT
= 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.97 Safari/537.36'
59 _VIDEO_PAGE_TEMPLATE
= 'https://www.facebook.com/video/video.php?v=%s'
60 _VIDEO_PAGE_TAHOE_TEMPLATE
= 'https://www.facebook.com/video/tahoe/async/%s/?chain=true&isvideo=true&payloadtype=primary'
63 'url': 'https://www.facebook.com/video.php?v=637842556329505&fref=nf',
64 'md5': '6a40d33c0eccbb1af76cf0485a052659',
66 'id': '637842556329505',
68 'title': 're:Did you know Kei Nishikori is the first Asian man to ever reach a Grand Slam',
69 'uploader': 'Tennis on Facebook',
70 'upload_date': '20140908',
71 'timestamp': 1410199200,
73 'skip': 'Requires logging in',
75 'url': 'https://www.facebook.com/video.php?v=274175099429670',
77 'id': '274175099429670',
79 'title': 're:^Asif Nawab Butt posted a video',
80 'uploader': 'Asif Nawab Butt',
81 'upload_date': '20140506',
82 'timestamp': 1399398998,
83 'thumbnail': r
're:^https?://.*',
85 'expected_warnings': [
89 'note': 'Video with DASH manifest',
90 'url': 'https://www.facebook.com/video.php?v=957955867617029',
91 'md5': 'b2c28d528273b323abe5c6ab59f0f030',
93 'id': '957955867617029',
95 'title': 'When you post epic content on instagram.com/433 8 million followers, this is ...',
96 'uploader': 'Demy de Zeeuw',
97 'upload_date': '20160110',
98 'timestamp': 1452431627,
100 'skip': 'Requires logging in',
102 'url': 'https://www.facebook.com/maxlayn/posts/10153807558977570',
103 'md5': '037b1fa7f3c2d02b7a0d7bc16031ecc6',
105 'id': '544765982287235',
107 'title': '"What are you doing running in the snow?"',
108 'uploader': 'FailArmy',
110 'skip': 'Video gone',
112 'url': 'https://m.facebook.com/story.php?story_fbid=1035862816472149&id=116132035111903',
113 'md5': '1deb90b6ac27f7efcf6d747c8a27f5e3',
115 'id': '1035862816472149',
117 'title': 'What the Flock Is Going On In New Zealand Credit: ViralHog',
118 'uploader': 'S. Saint',
120 'skip': 'Video gone',
122 'note': 'swf params escaped',
123 'url': 'https://www.facebook.com/barackobama/posts/10153664894881749',
124 'md5': '97ba073838964d12c70566e0085c2b91',
126 'id': '10153664894881749',
128 'title': 'Average time to confirm recent Supreme Court nominees: 67 days Longest it\'s t...',
129 'thumbnail': r
're:^https?://.*',
130 'timestamp': 1456259628,
131 'upload_date': '20160223',
132 'uploader': 'Barack Obama',
135 # have 1080P, but only up to 720p in swf params
136 'url': 'https://www.facebook.com/cnn/videos/10155529876156509/',
137 'md5': '9571fae53d4165bbbadb17a94651dcdc',
139 'id': '10155529876156509',
141 'title': 'She survived the holocaust — and years later, she’s getting her citizenship s...',
142 'timestamp': 1477818095,
143 'upload_date': '20161030',
145 'thumbnail': r
're:^https?://.*',
149 # bigPipe.onPageletArrive ... onPageletArrive pagelet_group_mall
150 'url': 'https://www.facebook.com/yaroslav.korpan/videos/1417995061575415/',
152 'id': '1417995061575415',
154 'title': 'md5:1db063d6a8c13faa8da727817339c857',
155 'timestamp': 1486648217,
156 'upload_date': '20170209',
157 'uploader': 'Yaroslav Korpan',
160 'skip_download': True,
163 'url': 'https://www.facebook.com/LaGuiaDelVaron/posts/1072691702860471',
165 'id': '1072691702860471',
167 'title': 'md5:ae2d22a93fbb12dad20dc393a869739d',
168 'timestamp': 1477305000,
169 'upload_date': '20161024',
170 'uploader': 'La Guía Del Varón',
171 'thumbnail': r
're:^https?://.*',
174 'skip_download': True,
177 'url': 'https://www.facebook.com/groups/1024490957622648/permalink/1396382447100162/',
179 'id': '1396382447100162',
181 'title': 'md5:19a428bbde91364e3de815383b54a235',
182 'timestamp': 1486035494,
183 'upload_date': '20170202',
184 'uploader': 'Elisabeth Ahtn',
187 'skip_download': True,
190 'url': 'https://www.facebook.com/video.php?v=10204634152394104',
191 'only_matching': True,
193 'url': 'https://www.facebook.com/amogood/videos/1618742068337349/?fref=nf',
194 'only_matching': True,
196 'url': 'https://www.facebook.com/ChristyClarkForBC/videos/vb.22819070941/10153870694020942/?type=2&theater',
197 'only_matching': True,
199 'url': 'facebook:544765982287235',
200 'only_matching': True,
202 'url': 'https://www.facebook.com/groups/164828000315060/permalink/764967300301124/',
203 'only_matching': True,
205 'url': 'https://zh-hk.facebook.com/peoplespower/videos/1135894589806027/',
206 'only_matching': True,
208 'url': 'https://www.facebookcorewwwi.onion/video.php?v=274175099429670',
209 'only_matching': True,
212 'url': 'https://www.facebook.com/onlycleverentertainment/videos/1947995502095005/',
213 'only_matching': True,
215 'url': 'https://www.facebook.com/WatchESLOne/videos/359649331226507/',
217 'id': '359649331226507',
219 'title': '#ESLOne VoD - Birmingham Finals Day#1 Fnatic vs. @Evil Geniuses',
220 'uploader': 'ESL One Dota 2',
223 'skip_download': True,
228 def _extract_urls(webpage
):
230 for mobj
in re
.finditer(
231 r
'<iframe[^>]+?src=(["\'])(?P
<url
>https?
://www\
.facebook\
.com
/(?
:video
/embed|plugins
/video\
.php
).+?
)\
1',
233 urls.append(mobj.group('url
'))
235 # see https://developers.facebook.com/docs/plugins/embedded-video-player
236 for mobj in re.finditer(r'''(?x)<div[^>]+
237 class=(?P<q1>[\'"])[^\'"]*\bfb-(?:video|post)\b[^\'"]*(?P=q1)[^>]+
238 data-href=(?P<q2>[\'"])(?P<url>(?:https?:)?//(?:www\.)?facebook.com/.+?)(?P=q2)''', webpage):
239 urls.append(mobj.group('url
'))
243 useremail, password = self._get_login_info()
244 if useremail is None:
247 login_page_req = sanitized_Request(self._LOGIN_URL)
248 self._set_cookie('facebook
.com
', 'locale
', 'en_US
')
249 login_page = self._download_webpage(login_page_req, None,
250 note='Downloading login page
',
251 errnote='Unable to download login page
')
252 lsd = self._search_regex(
253 r'<input type="hidden" name
="lsd" value
="([^"]*)"',
255 lgnrnd = self._search_regex(r'name="lgnrnd
" value="([^
"]*?)"', login_page, 'lgnrnd
')
262 'next
': 'http
://facebook
.com
/home
.php
',
263 'default_persistent
': '0',
264 'legacy_return
': '1',
268 request = sanitized_Request(self._LOGIN_URL, urlencode_postdata(login_form))
269 request.add_header('Content
-Type
', 'application
/x
-www
-form
-urlencoded
')
271 login_results = self._download_webpage(request, None,
272 note='Logging
in', errnote='unable to fetch login page
')
273 if re.search(r'<form(.*)name
="login"(.*)</form
>', login_results) is not None:
274 error = self._html_search_regex(
275 r'(?s
)<div
[^
>]+class=(["\']).*?login_error_box.*?\1[^>]*><div[^>]*>.*?</div><div[^>]*>(?P<error>.+?)</div>',
276 login_results, 'login error', default=None, group='error')
278 raise ExtractorError('Unable to login: %s' % error, expected=True)
279 self._downloader.report_warning('unable to log in: bad username/password, or exceeded login rate limit (~3/min). Check credentials or wait.')
282 fb_dtsg = self._search_regex(
283 r'name="fb_dtsg
" value="(.+?
)"', login_results, 'fb_dtsg', default=None)
284 h = self._search_regex(
285 r'name="h
"\s+(?:\w+="[^
"]+"\s
+)*?value
="([^"]+)"', login_results, 'h', default=None)
287 if not fb_dtsg or not h:
293 'name_action_selected': 'dont_save',
295 check_req = sanitized_Request(self._CHECKPOINT_URL, urlencode_postdata(check_form))
296 check_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
297 check_response = self._download_webpage(check_req, None,
298 note='Confirming login')
299 if re.search(r'id="checkpointSubmitButton
"', check_response) is not None:
300 self._downloader.report_warning('Unable to confirm login, you have to login in your browser and authorize the login.')
301 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
302 self._downloader.report_warning('unable to log in: %s' % error_to_compat_str(err))
305 def _real_initialize(self):
308 def _extract_from_url(self, url, video_id, fatal_if_no_video=True):
309 req = sanitized_Request(url)
310 req.add_header('User-Agent', self._CHROME_USER_AGENT)
311 webpage = self._download_webpage(req, video_id)
315 def extract_video_data(instances):
316 for item in instances:
317 if item[1][0] == 'VideoConfig':
318 video_item = item[2][0]
319 if video_item.get('video_id'):
320 return video_item['videoData']
322 server_js_data = self._parse_json(self._search_regex(
323 r'handleServerJS\(({.+})(?:\);|,")', webpage,
324 'server js data
', default='{}'), video_id, fatal=False)
327 video_data = extract_video_data(server_js_data.get('instances
', []))
329 def extract_from_jsmods_instances(js_data):
331 return extract_video_data(try_get(
332 js_data, lambda x: x['jsmods
']['instances
'], list) or [])
335 server_js_data = self._parse_json(
337 r'bigPipe\
.onPageletArrive\
(({.+?}
)\
)\s
*;\s
*}\s
*\
)\s
*,\s
*["\']onPageletArrive\s+(?:stream_pagelet|pagelet_group_mall|permalink_video_pagelet)',
338 webpage, 'js data', default='{}'),
339 video_id, transform_source=js_to_json, fatal=False)
340 video_data = extract_from_jsmods_instances(server_js_data)
343 if not fatal_if_no_video:
344 return webpage, False
345 m_msg = re.search(r'class="[^
"]*uiInterstitialContent[^"]*"><div>(.*?)</div>', webpage)
346 if m_msg is not None:
347 raise ExtractorError(
348 'The video is not available, Facebook said: "%s"' % m_msg.group(1),
350 elif '>You must log in to continue' in webpage:
351 self.raise_login_required()
353 # Video info not in first request, do a secondary request using
354 # tahoe player specific URL
355 tahoe_data = self._download_webpage(
356 self._VIDEO_PAGE_TAHOE_TEMPLATE % video_id, video_id,
357 data=urlencode_postdata({
359 '__pc': self._search_regex(
360 r'pkg_cohort["\']\s
*:\s
*["\'](.+?)["\']', webpage,
361 'pkg cohort
', default='PHASED
:DEFAULT
'),
362 '__rev
': self._search_regex(
363 r'client_revision
["\']\s*:\s*(\d+),', webpage,
364 'client revision', default='3944515'),
365 'fb_dtsg': self._search_regex(
366 r'"DTSGInitialData
"\s*,\s*\[\]\s*,\s*{\s*"token
"\s*:\s*"([^
"]+)"',
367 webpage, 'dtsg token
', default=''),
370 'Content
-Type
': 'application
/x
-www
-form
-urlencoded
',
372 tahoe_js_data = self._parse_json(
374 r'for\s
+\
(\s
*;\s
*;\s
*\
)\s
*;(.+)', tahoe_data,
375 'tahoe js data
', default='{}'),
376 video_id, fatal=False)
377 video_data = extract_from_jsmods_instances(tahoe_js_data)
380 raise ExtractorError('Cannot parse data
')
384 format_id = f['stream_type
']
385 if f and isinstance(f, dict):
387 if not f or not isinstance(f, list):
389 for quality in ('sd
', 'hd
'):
390 for src_type in ('src
', 'src_no_ratelimit
'):
391 src = f[0].get('%s_%s' % (quality, src_type))
393 preference = -10 if format_id == 'progressive
' else 0
397 'format_id
': '%s_%s_%s' % (format_id, quality, src_type),
399 'preference
': preference,
401 dash_manifest = f[0].get('dash_manifest
')
403 formats.extend(self._parse_mpd_formats(
404 compat_etree_fromstring(compat_urllib_parse_unquote_plus(dash_manifest))))
406 raise ExtractorError('Cannot find video formats
')
408 self._sort_formats(formats)
410 video_title = self._html_search_regex(
411 r'<h2\s
+[^
>]*class="uiHeaderTitle"[^
>]*>([^
<]*)</h2
>', webpage,
412 'title
', default=None)
414 video_title = self._html_search_regex(
415 r'(?s
)<span
class="fbPhotosPhotoCaption".*?
id="fbPhotoPageCaption"><span
class="hasCaption">(.*?
)</span
>',
416 webpage, 'alternative title
', default=None)
418 video_title = self._html_search_meta(
419 'description
', webpage, 'title
', default=None)
421 video_title = limit_length(video_title, 80)
423 video_title = 'Facebook video
#%s' % video_id
424 uploader
= clean_html(get_element_by_id(
425 'fbPhotoPageAuthorName', webpage
)) or self
._search
_regex
(
426 r
'ownerName\s*:\s*"([^"]+)"', webpage
, 'uploader',
427 default
=None) or self
._og
_search
_title
(webpage
, fatal
=False)
428 timestamp
= int_or_none(self
._search
_regex
(
429 r
'<abbr[^>]+data-utime=["\'](\d
+)', webpage,
430 'timestamp
', default=None))
431 thumbnail = self._og_search_thumbnail(webpage)
433 view_count = parse_count(self._search_regex(
434 r'\bviewCount\s
*:\s
*["\']([\d,.]+)', webpage, 'view count',
439 'title': video_title,
441 'uploader': uploader,
442 'timestamp': timestamp,
443 'thumbnail': thumbnail,
444 'view_count': view_count,
447 return webpage, info_dict
449 def _real_extract(self, url):
450 video_id = self._match_id(url)
452 real_url = self._VIDEO_PAGE_TEMPLATE % video_id if url.startswith('facebook:') else url
453 webpage, info_dict = self._extract_from_url(real_url, video_id, fatal_if_no_video=False)
460 self.url_result('facebook:%s' % vid, FacebookIE.ie_key())
461 for vid in self._parse_json(
463 r'(["\'])video_ids\
1\s
*:\s
*(?P
<ids
>\
[.+?\
])',
464 webpage, 'video ids
', group='ids
'),
467 return self.playlist_result(entries, video_id)
469 _, info_dict = self._extract_from_url(
470 self._VIDEO_PAGE_TEMPLATE % video_id,
471 video_id, fatal_if_no_video=True)
475 class FacebookPluginsVideoIE(InfoExtractor):
476 _VALID_URL = r'https?
://(?
:[\w
-]+\
.)?facebook\
.com
/plugins
/video\
.php
\?.*?
\bhref
=(?P
<id>https
.+)'
479 'url
': 'https
://www
.facebook
.com
/plugins
/video
.php?href
=https
%3A
%2F
%2Fwww
.facebook
.com
%2Fgov
.sg
%2Fvideos
%2F10154383743583686
%2F
&show_text
=0&width
=560',
480 'md5
': '5954e92cdfe51fe5782ae9bda7058a07
',
482 'id': '10154383743583686',
484 'title
': 'What to do during the haze?
',
485 'uploader
': 'Gov
.sg
',
486 'upload_date
': '20160826',
487 'timestamp
': 1472184808,
489 'add_ie
': [FacebookIE.ie_key()],
491 'url
': 'https
://www
.facebook
.com
/plugins
/video
.php?href
=https
%3A
%2F
%2Fwww
.facebook
.com
%2Fvideo
.php
%3Fv
%3D10204634152394104
',
492 'only_matching
': True,
494 'url
': 'https
://www
.facebook
.com
/plugins
/video
.php?href
=https
://www
.facebook
.com
/gov
.sg
/videos
/10154383743583686/&show_text
=0&width
=560',
495 'only_matching
': True,
498 def _real_extract(self, url):
499 return self.url_result(
500 compat_urllib_parse_unquote(self._match_id(url)),