[yt-dlp.git] / youtube_dl / extractor / facebook.py

from __future__ import unicode_literals

import re
import socket

from .common import InfoExtractor
from ..compat import (
    compat_etree_fromstring,
    compat_http_client,
    compat_urllib_error,
    compat_urllib_parse_unquote,
    compat_urllib_parse_unquote_plus,
)
from ..utils import (
    clean_html,
    error_to_compat_str,
    ExtractorError,
    get_element_by_id,
    int_or_none,
    js_to_json,
    limit_length,
    sanitized_Request,
    try_get,
    urlencode_postdata,
)


class FacebookIE(InfoExtractor):
    _VALID_URL = r'''(?x)
                (?:
                    https?://
                        (?:[\w-]+\.)?(?:facebook\.com|facebookcorewwwi\.onion)/
                        (?:[^#]*?\#!/)?
                        (?:
                            (?:
                                video/video\.php|
                                photo\.php|
                                video\.php|
                                video/embed|
                                story\.php
                            )\?(?:.*?)(?:v|video_id|story_fbid)=|
                            [^/]+/videos/(?:[^/]+/)?|
                            [^/]+/posts/|
                            groups/[^/]+/permalink/
                        )|
                    facebook:
                )
                (?P<id>[0-9]+)
                '''
    _LOGIN_URL = 'https://www.facebook.com/login.php?next=http%3A%2F%2Ffacebook.com%2Fhome.php&login_attempt=1'
    _CHECKPOINT_URL = 'https://www.facebook.com/checkpoint/?next=http%3A%2F%2Ffacebook.com%2Fhome.php&_fb_noscript=1'
    _NETRC_MACHINE = 'facebook'
    IE_NAME = 'facebook'

    _CHROME_USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.97 Safari/537.36'

    _VIDEO_PAGE_TEMPLATE = 'https://www.facebook.com/video/video.php?v=%s'

    _TESTS = [{
        'url': 'https://www.facebook.com/video.php?v=637842556329505&fref=nf',
        'md5': '6a40d33c0eccbb1af76cf0485a052659',
        'info_dict': {
            'id': '637842556329505',
            'ext': 'mp4',
            'title': 're:Did you know Kei Nishikori is the first Asian man to ever reach a Grand Slam',
            'uploader': 'Tennis on Facebook',
            'upload_date': '20140908',
            'timestamp': 1410199200,
        }
    }, {
        'note': 'Video without discernible title',
        'url': 'https://www.facebook.com/video.php?v=274175099429670',
        'info_dict': {
            'id': '274175099429670',
            'ext': 'mp4',
            'title': 'Asif Nawab Butt posted a video to his Timeline.',
            'uploader': 'Asif Nawab Butt',
            'upload_date': '20140506',
            'timestamp': 1399398998,
        },
        'expected_warnings': [
            'title'
        ]
    }, {
        'note': 'Video with DASH manifest',
        'url': 'https://www.facebook.com/video.php?v=957955867617029',
        'md5': 'b2c28d528273b323abe5c6ab59f0f030',
        'info_dict': {
            'id': '957955867617029',
            'ext': 'mp4',
            'title': 'When you post epic content on instagram.com/433 8 million followers, this is ...',
            'uploader': 'Demy de Zeeuw',
            'upload_date': '20160110',
            'timestamp': 1452431627,
        },
    }, {
        'url': 'https://www.facebook.com/maxlayn/posts/10153807558977570',
        'md5': '037b1fa7f3c2d02b7a0d7bc16031ecc6',
        'info_dict': {
            'id': '544765982287235',
            'ext': 'mp4',
            'title': '"What are you doing running in the snow?"',
            'uploader': 'FailArmy',
        },
        'skip': 'Video gone',
    }, {
        'url': 'https://m.facebook.com/story.php?story_fbid=1035862816472149&id=116132035111903',
        'md5': '1deb90b6ac27f7efcf6d747c8a27f5e3',
        'info_dict': {
            'id': '1035862816472149',
            'ext': 'mp4',
            'title': 'What the Flock Is Going On In New Zealand  Credit: ViralHog',
            'uploader': 'S. Saint',
        },
        'skip': 'Video gone',
    }, {
        'note': 'swf params escaped',
        'url': 'https://www.facebook.com/barackobama/posts/10153664894881749',
        'md5': '97ba073838964d12c70566e0085c2b91',
        'info_dict': {
            'id': '10153664894881749',
            'ext': 'mp4',
            'title': 'Facebook video #10153664894881749',
        },
    }, {
        # have 1080P, but only up to 720p in swf params
        'url': 'https://www.facebook.com/cnn/videos/10155529876156509/',
        'md5': '0d9813160b146b3bc8744e006027fcc6',
        'info_dict': {
            'id': '10155529876156509',
            'ext': 'mp4',
            'title': 'Holocaust survivor becomes US citizen',
            'timestamp': 1477818095,
            'upload_date': '20161030',
            'uploader': 'CNN',
        },
    }, {
        'url': 'https://www.facebook.com/video.php?v=10204634152394104',
        'only_matching': True,
    }, {
        'url': 'https://www.facebook.com/amogood/videos/1618742068337349/?fref=nf',
        'only_matching': True,
    }, {
        'url': 'https://www.facebook.com/ChristyClarkForBC/videos/vb.22819070941/10153870694020942/?type=2&theater',
        'only_matching': True,
    }, {
        'url': 'facebook:544765982287235',
        'only_matching': True,
    }, {
        'url': 'https://www.facebook.com/groups/164828000315060/permalink/764967300301124/',
        'only_matching': True,
    }, {
        'url': 'https://zh-hk.facebook.com/peoplespower/videos/1135894589806027/',
        'only_matching': True,
    }, {
        'url': 'https://www.facebookcorewwwi.onion/video.php?v=274175099429670',
        'only_matching': True,
    }]

    @staticmethod
    def _extract_url(webpage):
        mobj = re.search(
            r'<iframe[^>]+?src=(["\'])(?P<url>https://www\.facebook\.com/video/embed.+?)\1', webpage)
        if mobj is not None:
            return mobj.group('url')

        # Facebook API embed
        # see https://developers.facebook.com/docs/plugins/embedded-video-player
        mobj = re.search(r'''(?x)<div[^>]+
                class=(?P<q1>[\'"])[^\'"]*\bfb-(?:video|post)\b[^\'"]*(?P=q1)[^>]+
                data-href=(?P<q2>[\'"])(?P<url>(?:https?:)?//(?:www\.)?facebook.com/.+?)(?P=q2)''', webpage)
        if mobj is not None:
            return mobj.group('url')

    def _login(self):
        (useremail, password) = self._get_login_info()
        if useremail is None:
            return

        login_page_req = sanitized_Request(self._LOGIN_URL)
        self._set_cookie('facebook.com', 'locale', 'en_US')
        login_page = self._download_webpage(login_page_req, None,
                                            note='Downloading login page',
                                            errnote='Unable to download login page')
        lsd = self._search_regex(
            r'<input type="hidden" name="lsd" value="([^"]*)"',
            login_page, 'lsd')
        lgnrnd = self._search_regex(r'name="lgnrnd" value="([^"]*?)"', login_page, 'lgnrnd')

        login_form = {
            'email': useremail,
            'pass': password,
            'lsd': lsd,
            'lgnrnd': lgnrnd,
            'next': 'http://facebook.com/home.php',
            'default_persistent': '0',
            'legacy_return': '1',
            'timezone': '-60',
            'trynum': '1',
        }
        request = sanitized_Request(self._LOGIN_URL, urlencode_postdata(login_form))
        request.add_header('Content-Type', 'application/x-www-form-urlencoded')
        try:
            login_results = self._download_webpage(request, None,
                                                   note='Logging in', errnote='unable to fetch login page')
            if re.search(r'<form(.*)name="login"(.*)</form>', login_results) is not None:
                error = self._html_search_regex(
                    r'(?s)<div[^>]+class=(["\']).*?login_error_box.*?\1[^>]*><div[^>]*>.*?</div><div[^>]*>(?P<error>.+?)</div>',
                    login_results, 'login error', default=None, group='error')
                if error:
                    raise ExtractorError('Unable to login: %s' % error, expected=True)
                self._downloader.report_warning('unable to log in: bad username/password, or exceeded login rate limit (~3/min). Check credentials or wait.')
                return

            fb_dtsg = self._search_regex(
                r'name="fb_dtsg" value="(.+?)"', login_results, 'fb_dtsg', default=None)
            h = self._search_regex(
                r'name="h"\s+(?:\w+="[^"]+"\s+)*?value="([^"]+)"', login_results, 'h', default=None)

            if not fb_dtsg or not h:
                return

            check_form = {
                'fb_dtsg': fb_dtsg,
                'h': h,
                'name_action_selected': 'dont_save',
            }
            check_req = sanitized_Request(self._CHECKPOINT_URL, urlencode_postdata(check_form))
            check_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
            check_response = self._download_webpage(check_req, None,
                                                    note='Confirming login')
            if re.search(r'id="checkpointSubmitButton"', check_response) is not None:
                self._downloader.report_warning('Unable to confirm login, you have to login in your browser and authorize the login.')
        except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
            self._downloader.report_warning('unable to log in: %s' % error_to_compat_str(err))
            return

    def _real_initialize(self):
        self._login()

    def _extract_from_url(self, url, video_id, fatal_if_no_video=True):
        req = sanitized_Request(url)
        req.add_header('User-Agent', self._CHROME_USER_AGENT)
        webpage = self._download_webpage(req, video_id)

        video_data = None

        def extract_video_data(instances):
            for item in instances:
                if item[1][0] == 'VideoConfig':
                    video_item = item[2][0]
                    if video_item.get('video_id') == video_id:
                        return video_item['videoData']

        server_js_data = self._parse_json(self._search_regex(
            r'handleServerJS\(({.+})(?:\);|,")', webpage,
            'server js data', default='{}'), video_id, fatal=False)

        if server_js_data:
            video_data = extract_video_data(server_js_data.get('instances', []))

        if not video_data:
            server_js_data = self._parse_json(
                self._search_regex(
                    r'bigPipe\.onPageletArrive\(({.+?})\)\s*;\s*}\s*\)\s*,\s*["\']onPageletArrive\s+stream_pagelet',
                    webpage, 'js data', default='{}'),
                video_id, transform_source=js_to_json, fatal=False)
            if server_js_data:
                video_data = extract_video_data(try_get(
                    server_js_data, lambda x: x['jsmods']['instances'],
                    list) or [])

        if not video_data:
            if not fatal_if_no_video:
                return webpage, False
            m_msg = re.search(r'class="[^"]*uiInterstitialContent[^"]*"><div>(.*?)</div>', webpage)
            if m_msg is not None:
                raise ExtractorError(
                    'The video is not available, Facebook said: "%s"' % m_msg.group(1),
                    expected=True)
            elif '>You must log in to continue' in webpage:
                self.raise_login_required()
            else:
                raise ExtractorError('Cannot parse data')

        formats = []
        for f in video_data:
            format_id = f['stream_type']
            if f and isinstance(f, dict):
                f = [f]
            if not f or not isinstance(f, list):
                continue
            for quality in ('sd', 'hd'):
                for src_type in ('src', 'src_no_ratelimit'):
                    src = f[0].get('%s_%s' % (quality, src_type))
                    if src:
                        preference = -10 if format_id == 'progressive' else 0
                        if quality == 'hd':
                            preference += 5
                        formats.append({
                            'format_id': '%s_%s_%s' % (format_id, quality, src_type),
                            'url': src,
                            'preference': preference,
                        })
            dash_manifest = f[0].get('dash_manifest')
            if dash_manifest:
                formats.extend(self._parse_mpd_formats(
                    compat_etree_fromstring(compat_urllib_parse_unquote_plus(dash_manifest))))
        if not formats:
            raise ExtractorError('Cannot find video formats')

        self._sort_formats(formats)

        video_title = self._html_search_regex(
            r'<h2\s+[^>]*class="uiHeaderTitle"[^>]*>([^<]*)</h2>', webpage, 'title',
            default=None)
        if not video_title:
            video_title = self._html_search_regex(
                r'(?s)<span class="fbPhotosPhotoCaption".*?id="fbPhotoPageCaption"><span class="hasCaption">(.*?)</span>',
                webpage, 'alternative title', default=None)
        if not video_title:
            video_title = self._html_search_meta(
                'description', webpage, 'title')
        if video_title:
            video_title = limit_length(video_title, 80)
        else:
            video_title = 'Facebook video #%s' % video_id
        uploader = clean_html(get_element_by_id(
            'fbPhotoPageAuthorName', webpage)) or self._search_regex(
            r'ownerName\s*:\s*"([^"]+)"', webpage, 'uploader', fatal=False)
        timestamp = int_or_none(self._search_regex(
            r'<abbr[^>]+data-utime=["\'](\d+)', webpage,
            'timestamp', default=None))

        info_dict = {
            'id': video_id,
            'title': video_title,
            'formats': formats,
            'uploader': uploader,
            'timestamp': timestamp,
        }

        return webpage, info_dict

    def _real_extract(self, url):
        video_id = self._match_id(url)

        real_url = self._VIDEO_PAGE_TEMPLATE % video_id if url.startswith('facebook:') else url
        webpage, info_dict = self._extract_from_url(real_url, video_id, fatal_if_no_video=False)

        if info_dict:
            return info_dict

        if '/posts/' in url:
            entries = [
                self.url_result('facebook:%s' % vid, FacebookIE.ie_key())
                for vid in self._parse_json(
                    self._search_regex(
                        r'(["\'])video_ids\1\s*:\s*(?P<ids>\[.+?\])',
                        webpage, 'video ids', group='ids'),
                    video_id)]

            return self.playlist_result(entries, video_id)
        else:
            _, info_dict = self._extract_from_url(
                self._VIDEO_PAGE_TEMPLATE % video_id,
                video_id, fatal_if_no_video=True)
            return info_dict


class FacebookPluginsVideoIE(InfoExtractor):
    _VALID_URL = r'https?://(?:[\w-]+\.)?facebook\.com/plugins/video\.php\?.*?\bhref=(?P<id>https.+)'

    _TESTS = [{
        'url': 'https://www.facebook.com/plugins/video.php?href=https%3A%2F%2Fwww.facebook.com%2Fgov.sg%2Fvideos%2F10154383743583686%2F&show_text=0&width=560',
        'md5': '5954e92cdfe51fe5782ae9bda7058a07',
        'info_dict': {
            'id': '10154383743583686',
            'ext': 'mp4',
            'title': 'What to do during the haze?',
            'uploader': 'Gov.sg',
            'upload_date': '20160826',
            'timestamp': 1472184808,
        },
        'add_ie': [FacebookIE.ie_key()],
    }, {
        'url': 'https://www.facebook.com/plugins/video.php?href=https%3A%2F%2Fwww.facebook.com%2Fvideo.php%3Fv%3D10204634152394104',
        'only_matching': True,
    }, {
        'url': 'https://www.facebook.com/plugins/video.php?href=https://www.facebook.com/gov.sg/videos/10154383743583686/&show_text=0&width=560',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        return self.url_result(
            compat_urllib_parse_unquote(self._match_id(url)),
            FacebookIE.ie_key())
Commit	Line	Data
cb3bb2cf PH	1	from __future__ import unicode_literals
cb3bb2cf PH	2
9eae41dd PH	3	import re
	4	import socket
	5
	6	from .common import InfoExtractor
8c25f81b	7	from ..compat import (
0803753f	8	compat_etree_fromstring,
9eae41dd	9	compat_http_client,
9eae41dd	10	compat_urllib_error,
d7011316	11	compat_urllib_parse_unquote,
0803753f	12	compat_urllib_parse_unquote_plus,
8c25f81b PH	13	)
8c25f81b PH	14	from ..utils import (
b83ef507	15	clean_html,
9b9c5355	16	error_to_compat_str,
9eae41dd	17	ExtractorError,
b83ef507	18	get_element_by_id,
196c6ba0	19	int_or_none,
b83ef507	20	js_to_json,
a020a0dc	21	limit_length,
5c2266df	22	sanitized_Request,
b83ef507	23	try_get,
2fc9f2b4	24	urlencode_postdata,
9eae41dd PH	25	)
	26
	27
	28	class FacebookIE(InfoExtractor):
9834872b	29	_VALID_URL = r'''(?x)
2e6e742c S	30	(?:
2e6e742c S	31	https?://
30918999	32	(?:[\w-]+\.)?(?:facebook\.com\|facebookcorewwwi\.onion)/
2e6e742c S	33	(?:[^#]*?\#!/)?
	34	(?:
	35	(?:
	36	video/video\.php\|
	37	photo\.php\|
	38	video\.php\|
efbeddea YCH	39	video/embed\|
	40	story\.php
	41	)\?(?:.*?)(?:v\|video_id\|story_fbid)=\|
9c7b509b	42	[^/]+/videos/(?:[^/]+/)?\|
e738e433 YCH	43	[^/]+/posts/\|
e738e433 YCH	44	groups/[^/]+/permalink/
2e6e742c S	45	)\|
	46	facebook:
	47	)
	48	(?P<id>[0-9]+)
	49	'''
67874aef JMF	50	_LOGIN_URL = 'https://www.facebook.com/login.php?next=http%3A%2F%2Ffacebook.com%2Fhome.php&login_attempt=1'
67874aef JMF	51	_CHECKPOINT_URL = 'https://www.facebook.com/checkpoint/?next=http%3A%2F%2Ffacebook.com%2Fhome.php&_fb_noscript=1'
9eae41dd	52	_NETRC_MACHINE = 'facebook'
cb3bb2cf	53	IE_NAME = 'facebook'
0803753f YCH	54
	55	_CHROME_USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.97 Safari/537.36'
	56
9c7b509b YCH	57	_VIDEO_PAGE_TEMPLATE = 'https://www.facebook.com/video/video.php?v=%s'
9c7b509b YCH	58
60ac04e5	59	_TESTS = [{
50317dbb PH	60	'url': 'https://www.facebook.com/video.php?v=637842556329505&fref=nf',
50317dbb PH	61	'md5': '6a40d33c0eccbb1af76cf0485a052659',
cb3bb2cf	62	'info_dict': {
50317dbb	63	'id': '637842556329505',
cb3bb2cf	64	'ext': 'mp4',
5e4f0619	65	'title': 're:Did you know Kei Nishikori is the first Asian man to ever reach a Grand Slam',
67b8a28a	66	'uploader': 'Tennis on Facebook',
196c6ba0 S	67	'upload_date': '20140908',
196c6ba0 S	68	'timestamp': 1410199200,
6f5ac90c	69	}
a020a0dc PH	70	}, {
	71	'note': 'Video without discernible title',
	72	'url': 'https://www.facebook.com/video.php?v=274175099429670',
	73	'info_dict': {
	74	'id': '274175099429670',
	75	'ext': 'mp4',
5a116e13	76	'title': 'Asif Nawab Butt posted a video to his Timeline.',
67b8a28a	77	'uploader': 'Asif Nawab Butt',
196c6ba0 S	78	'upload_date': '20140506',
196c6ba0 S	79	'timestamp': 1399398998,
db3ca364 YCH	80	},
	81	'expected_warnings': [
	82	'title'
	83	]
0803753f YCH	84	}, {
	85	'note': 'Video with DASH manifest',
	86	'url': 'https://www.facebook.com/video.php?v=957955867617029',
196c6ba0	87	'md5': 'b2c28d528273b323abe5c6ab59f0f030',
0803753f YCH	88	'info_dict': {
	89	'id': '957955867617029',
	90	'ext': 'mp4',
	91	'title': 'When you post epic content on instagram.com/433 8 million followers, this is ...',
	92	'uploader': 'Demy de Zeeuw',
196c6ba0 S	93	'upload_date': '20160110',
196c6ba0 S	94	'timestamp': 1452431627,
0803753f	95	},
9c7b509b YCH	96	}, {
	97	'url': 'https://www.facebook.com/maxlayn/posts/10153807558977570',
	98	'md5': '037b1fa7f3c2d02b7a0d7bc16031ecc6',
	99	'info_dict': {
	100	'id': '544765982287235',
	101	'ext': 'mp4',
	102	'title': '"What are you doing running in the snow?"',
	103	'uploader': 'FailArmy',
d9ee2e5c YCH	104	},
d9ee2e5c YCH	105	'skip': 'Video gone',
98801241 YCH	106	}, {
	107	'url': 'https://m.facebook.com/story.php?story_fbid=1035862816472149&id=116132035111903',
	108	'md5': '1deb90b6ac27f7efcf6d747c8a27f5e3',
	109	'info_dict': {
	110	'id': '1035862816472149',
	111	'ext': 'mp4',
	112	'title': 'What the Flock Is Going On In New Zealand Credit: ViralHog',
	113	'uploader': 'S. Saint',
	114	},
d9ee2e5c	115	'skip': 'Video gone',
5080cbf9 YCH	116	}, {
	117	'note': 'swf params escaped',
	118	'url': 'https://www.facebook.com/barackobama/posts/10153664894881749',
	119	'md5': '97ba073838964d12c70566e0085c2b91',
	120	'info_dict': {
	121	'id': '10153664894881749',
	122	'ext': 'mp4',
	123	'title': 'Facebook video #10153664894881749',
	124	},
d9ee2e5c YCH	125	}, {
	126	# have 1080P, but only up to 720p in swf params
	127	'url': 'https://www.facebook.com/cnn/videos/10155529876156509/',
	128	'md5': '0d9813160b146b3bc8744e006027fcc6',
	129	'info_dict': {
	130	'id': '10155529876156509',
	131	'ext': 'mp4',
	132	'title': 'Holocaust survivor becomes US citizen',
	133	'timestamp': 1477818095,
	134	'upload_date': '20161030',
	135	'uploader': 'CNN',
	136	},
60ac04e5 PH	137	}, {
	138	'url': 'https://www.facebook.com/video.php?v=10204634152394104',
	139	'only_matching': True,
c6256697 YCH	140	}, {
	141	'url': 'https://www.facebook.com/amogood/videos/1618742068337349/?fref=nf',
	142	'only_matching': True,
53faa3ca YCH	143	}, {
	144	'url': 'https://www.facebook.com/ChristyClarkForBC/videos/vb.22819070941/10153870694020942/?type=2&theater',
	145	'only_matching': True,
2e6e742c S	146	}, {
	147	'url': 'facebook:544765982287235',
	148	'only_matching': True,
e738e433 YCH	149	}, {
	150	'url': 'https://www.facebook.com/groups/164828000315060/permalink/764967300301124/',
	151	'only_matching': True,
b4a131e1 S	152	}, {
	153	'url': 'https://zh-hk.facebook.com/peoplespower/videos/1135894589806027/',
	154	'only_matching': True,
30918999 YCH	155	}, {
	156	'url': 'https://www.facebookcorewwwi.onion/video.php?v=274175099429670',
	157	'only_matching': True,
60ac04e5	158	}]
9eae41dd	159
fd6ca382 YCH	160	@staticmethod
	161	def _extract_url(webpage):
	162	mobj = re.search(
	163	r'<iframe[^>]+?src=(["\'])(?P<url>https://www\.facebook\.com/video/embed.+?)\1', webpage)
	164	if mobj is not None:
	165	return mobj.group('url')
	166
	167	# Facebook API embed
	168	# see https://developers.facebook.com/docs/plugins/embedded-video-player
	169	mobj = re.search(r'''(?x)<div[^>]+
369bb062	170	class=(?P<q1>[\'"])[^\'"]\bfb-(?:video\|post)\b[^\'"](?P=q1)[^>]+
6a424391	171	data-href=(?P<q2>[\'"])(?P<url>(?:https?:)?//(?:www\.)?facebook.com/.+?)(?P=q2)''', webpage)
fd6ca382 YCH	172	if mobj is not None:
	173	return mobj.group('url')
	174
67874aef JMF	175	def _login(self):
67874aef JMF	176	(useremail, password) = self._get_login_info()
9eae41dd PH	177	if useremail is None:
	178	return
	179
5c2266df	180	login_page_req = sanitized_Request(self._LOGIN_URL)
c1f49e16	181	self._set_cookie('facebook.com', 'locale', 'en_US')
b74fa8cd	182	login_page = self._download_webpage(login_page_req, None,
9e1a5b84 JW	183	note='Downloading login page',
9e1a5b84 JW	184	errnote='Unable to download login page')
b37b9450	185	lsd = self._search_regex(
d0ff8384	186	r'<input type="hidden" name="lsd" value="([^"]*)"',
b37b9450	187	login_page, 'lsd')
cb3bb2cf	188	lgnrnd = self._search_regex(r'name="lgnrnd" value="([^"]*?)"', login_page, 'lgnrnd')
67874aef	189
9eae41dd PH	190	login_form = {
	191	'email': useremail,
	192	'pass': password,
67874aef JMF	193	'lsd': lsd,
	194	'lgnrnd': lgnrnd,
	195	'next': 'http://facebook.com/home.php',
	196	'default_persistent': '0',
	197	'legacy_return': '1',
	198	'timezone': '-60',
	199	'trynum': '1',
b74e86f4	200	}
5c2266df	201	request = sanitized_Request(self._LOGIN_URL, urlencode_postdata(login_form))
67874aef	202	request.add_header('Content-Type', 'application/x-www-form-urlencoded')
9eae41dd	203	try:
b74fa8cd	204	login_results = self._download_webpage(request, None,
9e1a5b84	205	note='Logging in', errnote='unable to fetch login page')
9eae41dd	206	if re.search(r'<form(.)name="login"(.)</form>', login_results) is not None:
9f66931e S	207	error = self._html_search_regex(
	208	r'(?s)<div[^>]+class=(["\']).?login_error_box.?\1[^>]><div[^>]>.?</div><div[^>]>(?P<error>.+?)</div>',
	209	login_results, 'login error', default=None, group='error')
	210	if error:
	211	raise ExtractorError('Unable to login: %s' % error, expected=True)
dfb1b146	212	self._downloader.report_warning('unable to log in: bad username/password, or exceeded login rate limit (~3/min). Check credentials or wait.')
9eae41dd	213	return
67874aef	214
c1f49e16 S	215	fb_dtsg = self._search_regex(
	216	r'name="fb_dtsg" value="(.+?)"', login_results, 'fb_dtsg', default=None)
	217	h = self._search_regex(
	218	r'name="h"\s+(?:\w+="[^"]+"\s+)*?value="([^"]+)"', login_results, 'h', default=None)
	219
	220	if not fb_dtsg or not h:
	221	return
	222
67874aef	223	check_form = {
c1f49e16 S	224	'fb_dtsg': fb_dtsg,
c1f49e16 S	225	'h': h,
67874aef	226	'name_action_selected': 'dont_save',
67874aef	227	}
5c2266df	228	check_req = sanitized_Request(self._CHECKPOINT_URL, urlencode_postdata(check_form))
67874aef	229	check_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
b74fa8cd	230	check_response = self._download_webpage(check_req, None,
9e1a5b84	231	note='Confirming login')
67874aef	232	if re.search(r'id="checkpointSubmitButton"', check_response) is not None:
dfb1b146	233	self._downloader.report_warning('Unable to confirm login, you have to login in your browser and authorize the login.')
9eae41dd	234	except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
9b9c5355	235	self._downloader.report_warning('unable to log in: %s' % error_to_compat_str(err))
9eae41dd PH	236	return
9eae41dd PH	237
67874aef JMF	238	def _real_initialize(self):
	239	self._login()
	240
9c7b509b YCH	241	def _extract_from_url(self, url, video_id, fatal_if_no_video=True):
9c7b509b YCH	242	req = sanitized_Request(url)
0803753f YCH	243	req.add_header('User-Agent', self._CHROME_USER_AGENT)
0803753f YCH	244	webpage = self._download_webpage(req, video_id)
9eae41dd	245
c1406299 YCH	246	video_data = None
c1406299 YCH	247
b83ef507 S	248	def extract_video_data(instances):
	249	for item in instances:
	250	if item[1][0] == 'VideoConfig':
	251	video_item = item[2][0]
	252	if video_item.get('video_id') == video_id:
	253	return video_item['videoData']
	254
d9ee2e5c	255	server_js_data = self._parse_json(self._search_regex(
b83ef507 S	256	r'handleServerJS\(({.+})(?:\);\|,")', webpage,
	257	'server js data', default='{}'), video_id, fatal=False)
	258
	259	if server_js_data:
	260	video_data = extract_video_data(server_js_data.get('instances', []))
	261
	262	if not video_data:
	263	server_js_data = self._parse_json(
	264	self._search_regex(
	265	r'bigPipe\.onPageletArrive\(({.+?})\)\s;\s}\s\)\s,\s*["\']onPageletArrive\s+stream_pagelet',
	266	webpage, 'js data', default='{}'),
	267	video_id, transform_source=js_to_json, fatal=False)
	268	if server_js_data:
	269	video_data = extract_video_data(try_get(
	270	server_js_data, lambda x: x['jsmods']['instances'],
	271	list) or [])
c1406299	272
c1406299	273	if not video_data:
9c7b509b YCH	274	if not fatal_if_no_video:
9c7b509b YCH	275	return webpage, False
6f71ef58 JMF	276	m_msg = re.search(r'class="[^"]uiInterstitialContent[^"]"><div>(.*?)</div>', webpage)
6f71ef58 JMF	277	if m_msg is not None:
749a4fd2	278	raise ExtractorError(
cb3bb2cf	279	'The video is not available, Facebook said: "%s"' % m_msg.group(1),
749a4fd2	280	expected=True)
d2d2495e S	281	elif '>You must log in to continue' in webpage:
d2d2495e S	282	self.raise_login_required()
6f71ef58	283	else:
cb3bb2cf	284	raise ExtractorError('Cannot parse data')
ffdf972b JMF	285
ffdf972b JMF	286	formats = []
d9ee2e5c YCH	287	for f in video_data:
d9ee2e5c YCH	288	format_id = f['stream_type']
ca74c90b SC	289	if f and isinstance(f, dict):
ca74c90b SC	290	f = [f]
44d6dd08 S	291	if not f or not isinstance(f, list):
	292	continue
	293	for quality in ('sd', 'hd'):
	294	for src_type in ('src', 'src_no_ratelimit'):
	295	src = f[0].get('%s_%s' % (quality, src_type))
	296	if src:
c24883a1 YCH	297	preference = -10 if format_id == 'progressive' else 0
	298	if quality == 'hd':
	299	preference += 5
44d6dd08 S	300	formats.append({
	301	'format_id': '%s_%s_%s' % (format_id, quality, src_type),
	302	'url': src,
c24883a1	303	'preference': preference,
44d6dd08	304	})
0803753f YCH	305	dash_manifest = f[0].get('dash_manifest')
0803753f YCH	306	if dash_manifest:
91cb6b50	307	formats.extend(self._parse_mpd_formats(
c11875b3	308	compat_etree_fromstring(compat_urllib_parse_unquote_plus(dash_manifest))))
ffdf972b JMF	309	if not formats:
ffdf972b JMF	310	raise ExtractorError('Cannot find video formats')
9eae41dd	311
0803753f YCH	312	self._sort_formats(formats)
0803753f YCH	313
81ec7c79	314	video_title = self._html_search_regex(
bf24c3d0	315	r'<h2\s+[^>]class="uiHeaderTitle"[^>]>([^<]*)</h2>', webpage, 'title',
db3ca364	316	default=None)
50317dbb PH	317	if not video_title:
	318	video_title = self._html_search_regex(
	319	r'(?s)<span class="fbPhotosPhotoCaption".?id="fbPhotoPageCaption"><span class="hasCaption">(.?)</span>',
59ee8a86	320	webpage, 'alternative title', default=None)
50317dbb	321	if not video_title:
5a116e13 S	322	video_title = self._html_search_meta(
	323	'description', webpage, 'title')
	324	if video_title:
	325	video_title = limit_length(video_title, 80)
	326	else:
50317dbb	327	video_title = 'Facebook video #%s' % video_id
5a116e13 S	328	uploader = clean_html(get_element_by_id(
	329	'fbPhotoPageAuthorName', webpage)) or self._search_regex(
	330	r'ownerName\s:\s"([^"]+)"', webpage, 'uploader', fatal=False)
196c6ba0 S	331	timestamp = int_or_none(self._search_regex(
	332	r'<abbr[^>]+data-utime=["\'](\d+)', webpage,
	333	'timestamp', default=None))
9eae41dd	334
9c7b509b	335	info_dict = {
9eae41dd PH	336	'id': video_id,
9eae41dd PH	337	'title': video_title,
ffdf972b	338	'formats': formats,
51a57515	339	'uploader': uploader,
196c6ba0	340	'timestamp': timestamp,
9eae41dd	341	}
de691a49	342
9c7b509b	343	return webpage, info_dict
de691a49 S	344
de691a49 S	345	def _real_extract(self, url):
9c7b509b YCH	346	video_id = self._match_id(url)
	347
	348	real_url = self._VIDEO_PAGE_TEMPLATE % video_id if url.startswith('facebook:') else url
	349	webpage, info_dict = self._extract_from_url(real_url, video_id, fatal_if_no_video=False)
de691a49	350
9c7b509b YCH	351	if info_dict:
9c7b509b YCH	352	return info_dict
de691a49	353
9c7b509b YCH	354	if '/posts/' in url:
9c7b509b YCH	355	entries = [
f6f6217a JMF	356	self.url_result('facebook:%s' % vid, FacebookIE.ie_key())
f6f6217a JMF	357	for vid in self._parse_json(
9c7b509b YCH	358	self._search_regex(
	359	r'(["\'])video_ids\1\s:\s(?P<ids>\[.+?\])',
	360	webpage, 'video ids', group='ids'),
	361	video_id)]
de691a49	362
9c7b509b YCH	363	return self.playlist_result(entries, video_id)
	364	else:
	365	_, info_dict = self._extract_from_url(
	366	self._VIDEO_PAGE_TEMPLATE % video_id,
	367	video_id, fatal_if_no_video=True)
	368	return info_dict
349fc5c7 S	369
	370
	371	class FacebookPluginsVideoIE(InfoExtractor):
	372	_VALID_URL = r'https?://(?:[\w-]+\.)?facebook\.com/plugins/video\.php\?.*?\bhref=(?P<id>https.+)'
	373
	374	_TESTS = [{
	375	'url': 'https://www.facebook.com/plugins/video.php?href=https%3A%2F%2Fwww.facebook.com%2Fgov.sg%2Fvideos%2F10154383743583686%2F&show_text=0&width=560',
	376	'md5': '5954e92cdfe51fe5782ae9bda7058a07',
	377	'info_dict': {
	378	'id': '10154383743583686',
	379	'ext': 'mp4',
	380	'title': 'What to do during the haze?',
	381	'uploader': 'Gov.sg',
	382	'upload_date': '20160826',
	383	'timestamp': 1472184808,
	384	},
	385	'add_ie': [FacebookIE.ie_key()],
	386	}, {
	387	'url': 'https://www.facebook.com/plugins/video.php?href=https%3A%2F%2Fwww.facebook.com%2Fvideo.php%3Fv%3D10204634152394104',
	388	'only_matching': True,
	389	}, {
	390	'url': 'https://www.facebook.com/plugins/video.php?href=https://www.facebook.com/gov.sg/videos/10154383743583686/&show_text=0&width=560',
	391	'only_matching': True,
	392	}]
	393
	394	def _real_extract(self, url):
	395	return self.url_result(
	396	compat_urllib_parse_unquote(self._match_id(url)),
	397	FacebookIE.ie_key())