[yt-dlp.git] / youtube_dl / extractor / twitter.py

# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import compat_urlparse
from ..utils import (
    determine_ext,
    dict_get,
    ExtractorError,
    float_or_none,
    int_or_none,
    remove_end,
    try_get,
    xpath_text,
)

from .periscope import PeriscopeIE


class TwitterBaseIE(InfoExtractor):
    def _extract_formats_from_vmap_url(self, vmap_url, video_id):
        vmap_data = self._download_xml(vmap_url, video_id)
        video_url = xpath_text(vmap_data, './/MediaFile').strip()
        if determine_ext(video_url) == 'm3u8':
            return self._extract_m3u8_formats(
                video_url, video_id, ext='mp4', m3u8_id='hls',
                entry_protocol='m3u8_native')
        return [{
            'url': video_url,
        }]

    @staticmethod
    def _search_dimensions_in_video_url(a_format, video_url):
        m = re.search(r'/(?P<width>\d+)x(?P<height>\d+)/', video_url)
        if m:
            a_format.update({
                'width': int(m.group('width')),
                'height': int(m.group('height')),
            })


class TwitterCardIE(TwitterBaseIE):
    IE_NAME = 'twitter:card'
    _VALID_URL = r'https?://(?:www\.)?twitter\.com/i/(?:cards/tfw/v1|videos(?:/tweet)?)/(?P<id>\d+)'
    _TESTS = [
        {
            'url': 'https://twitter.com/i/cards/tfw/v1/560070183650213889',
            # MD5 checksums are different in different places
            'info_dict': {
                'id': '560070183650213889',
                'ext': 'mp4',
                'title': 'Twitter Card',
                'thumbnail': r're:^https?://.*\.jpg$',
                'duration': 30.033,
            },
            'skip': 'Video gone',
        },
        {
            'url': 'https://twitter.com/i/cards/tfw/v1/623160978427936768',
            'md5': '7ee2a553b63d1bccba97fbed97d9e1c8',
            'info_dict': {
                'id': '623160978427936768',
                'ext': 'mp4',
                'title': 'Twitter Card',
                'thumbnail': r're:^https?://.*\.jpg',
                'duration': 80.155,
            },
            'skip': 'Video gone',
        },
        {
            'url': 'https://twitter.com/i/cards/tfw/v1/654001591733886977',
            'md5': 'b6d9683dd3f48e340ded81c0e917ad46',
            'info_dict': {
                'id': 'dq4Oj5quskI',
                'ext': 'mp4',
                'title': 'Ubuntu 11.10 Overview',
                'description': 'md5:a831e97fa384863d6e26ce48d1c43376',
                'upload_date': '20111013',
                'uploader': 'OMG! Ubuntu!',
                'uploader_id': 'omgubuntu',
            },
            'add_ie': ['Youtube'],
        },
        {
            'url': 'https://twitter.com/i/cards/tfw/v1/665289828897005568',
            'md5': '6dabeaca9e68cbb71c99c322a4b42a11',
            'info_dict': {
                'id': 'iBb2x00UVlv',
                'ext': 'mp4',
                'upload_date': '20151113',
                'uploader_id': '1189339351084113920',
                'uploader': 'ArsenalTerje',
                'title': 'Vine by ArsenalTerje',
                'timestamp': 1447451307,
            },
            'add_ie': ['Vine'],
        }, {
            'url': 'https://twitter.com/i/videos/tweet/705235433198714880',
            'md5': '884812a2adc8aaf6fe52b15ccbfa3b88',
            'info_dict': {
                'id': '705235433198714880',
                'ext': 'mp4',
                'title': 'Twitter web player',
                'thumbnail': r're:^https?://.*',
            },
        }, {
            'url': 'https://twitter.com/i/videos/752274308186120192',
            'only_matching': True,
        },
    ]

    def _parse_media_info(self, media_info, video_id):
        formats = []
        for media_variant in media_info.get('variants', []):
            media_url = media_variant['url']
            if media_url.endswith('.m3u8'):
                formats.extend(self._extract_m3u8_formats(media_url, video_id, ext='mp4', m3u8_id='hls'))
            elif media_url.endswith('.mpd'):
                formats.extend(self._extract_mpd_formats(media_url, video_id, mpd_id='dash'))
            else:
                vbr = int_or_none(dict_get(media_variant, ('bitRate', 'bitrate')), scale=1000)
                a_format = {
                    'url': media_url,
                    'format_id': 'http-%d' % vbr if vbr else 'http',
                    'vbr': vbr,
                }
                # Reported bitRate may be zero
                if not a_format['vbr']:
                    del a_format['vbr']

                self._search_dimensions_in_video_url(a_format, media_url)

                formats.append(a_format)
        return formats

    def _extract_mobile_formats(self, username, video_id):
        webpage = self._download_webpage(
            'https://mobile.twitter.com/%s/status/%s' % (username, video_id),
            video_id, 'Downloading mobile webpage',
            headers={
                # A recent mobile UA is necessary for `gt` cookie
                'User-Agent': 'Mozilla/5.0 (Android 6.0.1; Mobile; rv:54.0) Gecko/54.0 Firefox/54.0',
            })
        main_script_url = self._html_search_regex(
            r'<script[^>]+src="([^"]+main\.[^"]+)"', webpage, 'main script URL')
        main_script = self._download_webpage(
            main_script_url, video_id, 'Downloading main script')
        bearer_token = self._search_regex(
            r'BEARER_TOKEN\s*:\s*"([^"]+)"',
            main_script, 'bearer token')
        guest_token = self._search_regex(
            r'document\.cookie\s*=\s*decodeURIComponent\("gt=(\d+)',
            webpage, 'guest token')
        api_data = self._download_json(
            'https://api.twitter.com/2/timeline/conversation/%s.json' % video_id,
            video_id, 'Downloading mobile API data',
            headers={
                'Authorization': 'Bearer ' + bearer_token,
                'x-guest-token': guest_token,
            })
        media_info = try_get(api_data, lambda o: o['globalObjects']['tweets'][video_id]
                                                  ['extended_entities']['media'][0]['video_info']) or {}
        return self._parse_media_info(media_info, video_id)

    def _real_extract(self, url):
        video_id = self._match_id(url)

        config = None
        formats = []
        duration = None

        webpage = self._download_webpage(url, video_id)

        iframe_url = self._html_search_regex(
            r'<iframe[^>]+src="((?:https?:)?//(?:www.youtube.com/embed/[^"]+|(?:www\.)?vine\.co/v/\w+/card))"',
            webpage, 'video iframe', default=None)
        if iframe_url:
            return self.url_result(iframe_url)

        config = self._parse_json(self._html_search_regex(
            r'data-(?:player-)?config="([^"]+)"', webpage,
            'data player config', default='{}'),
            video_id)

        if config.get('source_type') == 'vine':
            return self.url_result(config['player_url'], 'Vine')

        periscope_url = PeriscopeIE._extract_url(webpage)
        if periscope_url:
            return self.url_result(periscope_url, PeriscopeIE.ie_key())

        video_url = config.get('video_url') or config.get('playlist', [{}])[0].get('source')

        if video_url:
            if determine_ext(video_url) == 'm3u8':
                formats.extend(self._extract_m3u8_formats(video_url, video_id, ext='mp4', m3u8_id='hls'))
            else:
                f = {
                    'url': video_url,
                }

                self._search_dimensions_in_video_url(f, video_url)

                formats.append(f)

        vmap_url = config.get('vmapUrl') or config.get('vmap_url')
        if vmap_url:
            formats.extend(
                self._extract_formats_from_vmap_url(vmap_url, video_id))

        media_info = None

        for entity in config.get('status', {}).get('entities', []):
            if 'mediaInfo' in entity:
                media_info = entity['mediaInfo']

        if media_info:
            formats.extend(self._parse_media_info(media_info, video_id))
            duration = float_or_none(media_info.get('duration', {}).get('nanos'), scale=1e9)

        username = config.get('user', {}).get('screen_name')
        if username:
            formats.extend(self._extract_mobile_formats(username, video_id))

        self._remove_duplicate_formats(formats)
        self._sort_formats(formats)

        title = self._search_regex(r'<title>([^<]+)</title>', webpage, 'title')
        thumbnail = config.get('posterImageUrl') or config.get('image_src')
        duration = float_or_none(config.get('duration')) or duration

        return {
            'id': video_id,
            'title': title,
            'thumbnail': thumbnail,
            'duration': duration,
            'formats': formats,
        }


class TwitterIE(InfoExtractor):
    IE_NAME = 'twitter'
    _VALID_URL = r'https?://(?:www\.|m\.|mobile\.)?twitter\.com/(?P<user_id>[^/]+)/status/(?P<id>\d+)'
    _TEMPLATE_URL = 'https://twitter.com/%s/status/%s'

    _TESTS = [{
        'url': 'https://twitter.com/freethenipple/status/643211948184596480',
        'info_dict': {
            'id': '643211948184596480',
            'ext': 'mp4',
            'title': 'FREE THE NIPPLE - FTN supporters on Hollywood Blvd today!',
            'thumbnail': r're:^https?://.*\.jpg',
            'description': 'FREE THE NIPPLE on Twitter: "FTN supporters on Hollywood Blvd today! http://t.co/c7jHH749xJ"',
            'uploader': 'FREE THE NIPPLE',
            'uploader_id': 'freethenipple',
        },
        'params': {
            'skip_download': True,  # requires ffmpeg
        },
    }, {
        'url': 'https://twitter.com/giphz/status/657991469417025536/photo/1',
        'md5': 'f36dcd5fb92bf7057f155e7d927eeb42',
        'info_dict': {
            'id': '657991469417025536',
            'ext': 'mp4',
            'title': 'Gifs - tu vai cai tu vai cai tu nao eh capaz disso tu vai cai',
            'description': 'Gifs on Twitter: "tu vai cai tu vai cai tu nao eh capaz disso tu vai cai https://t.co/tM46VHFlO5"',
            'thumbnail': r're:^https?://.*\.png',
            'uploader': 'Gifs',
            'uploader_id': 'giphz',
        },
        'expected_warnings': ['height', 'width'],
        'skip': 'Account suspended',
    }, {
        'url': 'https://twitter.com/starwars/status/665052190608723968',
        'md5': '39b7199856dee6cd4432e72c74bc69d4',
        'info_dict': {
            'id': '665052190608723968',
            'ext': 'mp4',
            'title': 'Star Wars - A new beginning is coming December 18. Watch the official 60 second #TV spot for #StarWars: #TheForceAwakens.',
            'description': 'Star Wars on Twitter: "A new beginning is coming December 18. Watch the official 60 second #TV spot for #StarWars: #TheForceAwakens."',
            'uploader_id': 'starwars',
            'uploader': 'Star Wars',
        },
    }, {
        'url': 'https://twitter.com/BTNBrentYarina/status/705235433198714880',
        'info_dict': {
            'id': '705235433198714880',
            'ext': 'mp4',
            'title': 'Brent Yarina - Khalil Iverson\'s missed highlight dunk. And made highlight dunk. In one highlight.',
            'description': 'Brent Yarina on Twitter: "Khalil Iverson\'s missed highlight dunk. And made highlight dunk. In one highlight."',
            'uploader_id': 'BTNBrentYarina',
            'uploader': 'Brent Yarina',
        },
        'params': {
            # The same video as https://twitter.com/i/videos/tweet/705235433198714880
            # Test case of TwitterCardIE
            'skip_download': True,
        },
    }, {
        'url': 'https://twitter.com/jaydingeer/status/700207533655363584',
        'md5': '',
        'info_dict': {
            'id': '700207533655363584',
            'ext': 'mp4',
            'title': 'Donte - BEAT PROD: @suhmeduh #Damndaniel',
            'description': 'Donte on Twitter: "BEAT PROD: @suhmeduh  https://t.co/HBrQ4AfpvZ #Damndaniel https://t.co/byBooq2ejZ"',
            'thumbnail': r're:^https?://.*\.jpg',
            'uploader': 'Donte',
            'uploader_id': 'jaydingeer',
        },
        'params': {
            'skip_download': True,  # requires ffmpeg
        },
    }, {
        'url': 'https://twitter.com/Filmdrunk/status/713801302971588609',
        'md5': '89a15ed345d13b86e9a5a5e051fa308a',
        'info_dict': {
            'id': 'MIOxnrUteUd',
            'ext': 'mp4',
            'title': 'FilmDrunk - Vine of the day',
            'description': 'FilmDrunk on Twitter: "Vine of the day https://t.co/xmTvRdqxWf"',
            'uploader': 'FilmDrunk',
            'uploader_id': 'Filmdrunk',
            'timestamp': 1402826626,
            'upload_date': '20140615',
        },
        'add_ie': ['Vine'],
    }, {
        'url': 'https://twitter.com/captainamerica/status/719944021058060289',
        'info_dict': {
            'id': '719944021058060289',
            'ext': 'mp4',
            'title': 'Captain America - @King0fNerd Are you sure you made the right choice? Find out in theaters.',
            'description': 'Captain America on Twitter: "@King0fNerd Are you sure you made the right choice? Find out in theaters. https://t.co/GpgYi9xMJI"',
            'uploader_id': 'captainamerica',
            'uploader': 'Captain America',
        },
        'params': {
            'skip_download': True,  # requires ffmpeg
        },
    }, {
        'url': 'https://twitter.com/OPP_HSD/status/779210622571536384',
        'info_dict': {
            'id': '1zqKVVlkqLaKB',
            'ext': 'mp4',
            'title': 'Sgt Kerry Schmidt - LIVE on #Periscope: Road rage, mischief, assault, rollover and fire in one occurrence',
            'description': 'Sgt Kerry Schmidt on Twitter: "LIVE on #Periscope: Road rage, mischief, assault, rollover and fire in one occurrence  https://t.co/EKrVgIXF3s"',
            'upload_date': '20160923',
            'uploader_id': 'OPP_HSD',
            'uploader': 'Sgt Kerry Schmidt',
            'timestamp': 1474613214,
        },
        'add_ie': ['Periscope'],
    }, {
        # has mp4 formats via mobile API
        'url': 'https://twitter.com/news_al3alm/status/852138619213144067',
        'info_dict': {
            'id': '852138619213144067',
            'ext': 'mp4',
            'title': 'عالم الأخبار - كلمة تاريخية بجلسة الجناسي التاريخية.. النائب خالد مؤنس العتيبي للمعارضين : اتقوا الله .. الظلم ظلمات يوم القيامة',
            'description': 'عالم الأخبار on Twitter: "كلمة تاريخية بجلسة الجناسي التاريخية.. النائب خالد مؤنس العتيبي للمعارضين : اتقوا الله .. الظلم ظلمات يوم القيامة   https://t.co/xg6OhpyKfN"',
            'uploader': 'عالم الأخبار',
            'uploader_id': 'news_al3alm',
        },
        'params': {
            'format': 'best[format_id^=http-]',
        },
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        user_id = mobj.group('user_id')
        twid = mobj.group('id')

        webpage, urlh = self._download_webpage_handle(
            self._TEMPLATE_URL % (user_id, twid), twid)

        if 'twitter.com/account/suspended' in urlh.geturl():
            raise ExtractorError('Account suspended by Twitter.', expected=True)

        username = remove_end(self._og_search_title(webpage), ' on Twitter')

        title = description = self._og_search_description(webpage).strip('').replace('\n', ' ').strip('“”')

        # strip  'https -_t.co_BJYgOjSeGA' junk from filenames
        title = re.sub(r'\s+(https?://[^ ]+)', '', title)

        info = {
            'uploader_id': user_id,
            'uploader': username,
            'webpage_url': url,
            'description': '%s on Twitter: "%s"' % (username, description),
            'title': username + ' - ' + title,
        }

        mobj = re.search(r'''(?x)
            <video[^>]+class="animated-gif"(?P<more_info>[^>]+)>\s*
                <source[^>]+video-src="(?P<url>[^"]+)"
        ''', webpage)

        if mobj:
            more_info = mobj.group('more_info')
            height = int_or_none(self._search_regex(
                r'data-height="(\d+)"', more_info, 'height', fatal=False))
            width = int_or_none(self._search_regex(
                r'data-width="(\d+)"', more_info, 'width', fatal=False))
            thumbnail = self._search_regex(
                r'poster="([^"]+)"', more_info, 'poster', fatal=False)
            info.update({
                'id': twid,
                'url': mobj.group('url'),
                'height': height,
                'width': width,
                'thumbnail': thumbnail,
            })
            return info

        twitter_card_url = None
        if 'class="PlayableMedia' in webpage:
            twitter_card_url = '%s//twitter.com/i/videos/tweet/%s' % (self.http_scheme(), twid)
        else:
            twitter_card_iframe_url = self._search_regex(
                r'data-full-card-iframe-url=([\'"])(?P<url>(?:(?!\1).)+)\1',
                webpage, 'Twitter card iframe URL', default=None, group='url')
            if twitter_card_iframe_url:
                twitter_card_url = compat_urlparse.urljoin(url, twitter_card_iframe_url)

        if twitter_card_url:
            info.update({
                '_type': 'url_transparent',
                'ie_key': 'TwitterCard',
                'url': twitter_card_url,
            })
            return info

        raise ExtractorError('There\'s no video in this tweet.')


class TwitterAmplifyIE(TwitterBaseIE):
    IE_NAME = 'twitter:amplify'
    _VALID_URL = r'https?://amp\.twimg\.com/v/(?P<id>[0-9a-f\-]{36})'

    _TEST = {
        'url': 'https://amp.twimg.com/v/0ba0c3c7-0af3-4c0a-bed5-7efd1ffa2951',
        'md5': '7df102d0b9fd7066b86f3159f8e81bf6',
        'info_dict': {
            'id': '0ba0c3c7-0af3-4c0a-bed5-7efd1ffa2951',
            'ext': 'mp4',
            'title': 'Twitter Video',
            'thumbnail': 're:^https?://.*',
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        vmap_url = self._html_search_meta(
            'twitter:amplify:vmap', webpage, 'vmap url')
        formats = self._extract_formats_from_vmap_url(vmap_url, video_id)

        thumbnails = []
        thumbnail = self._html_search_meta(
            'twitter:image:src', webpage, 'thumbnail', fatal=False)

        def _find_dimension(target):
            w = int_or_none(self._html_search_meta(
                'twitter:%s:width' % target, webpage, fatal=False))
            h = int_or_none(self._html_search_meta(
                'twitter:%s:height' % target, webpage, fatal=False))
            return w, h

        if thumbnail:
            thumbnail_w, thumbnail_h = _find_dimension('image')
            thumbnails.append({
                'url': thumbnail,
                'width': thumbnail_w,
                'height': thumbnail_h,
            })

        video_w, video_h = _find_dimension('player')
        formats[0].update({
            'width': video_w,
            'height': video_h,
        })

        return {
            'id': video_id,
            'title': 'Twitter Video',
            'formats': formats,
            'thumbnails': thumbnails,
        }
Commit	Line	Data
48aae2d2	1	# coding: utf-8
23e7cba8 S	2	from __future__ import unicode_literals
	3
	4	import re
	5
	6	from .common import InfoExtractor
f0bc5a86	7	from ..compat import compat_urlparse
23e7cba8	8	from ..utils import (
c6308b31	9	determine_ext,
2edfd745 YCH	10	dict_get,
2edfd745 YCH	11	ExtractorError,
23e7cba8	12	float_or_none,
cf5881fc	13	int_or_none,
2edfd745 YCH	14	remove_end,
	15	try_get,
	16	xpath_text,
23e7cba8 S	17	)
23e7cba8 S	18
f0bc5a86 YCH	19	from .periscope import PeriscopeIE
f0bc5a86 YCH	20
23e7cba8	21
445d72b8	22	class TwitterBaseIE(InfoExtractor):
9be31e77	23	def _extract_formats_from_vmap_url(self, vmap_url, video_id):
445d72b8	24	vmap_data = self._download_xml(vmap_url, video_id)
9be31e77 YCH	25	video_url = xpath_text(vmap_data, './/MediaFile').strip()
	26	if determine_ext(video_url) == 'm3u8':
	27	return self._extract_m3u8_formats(
	28	video_url, video_id, ext='mp4', m3u8_id='hls',
	29	entry_protocol='m3u8_native')
	30	return [{
	31	'url': video_url,
	32	}]
445d72b8	33
2edfd745 YCH	34	@staticmethod
	35	def _search_dimensions_in_video_url(a_format, video_url):
	36	m = re.search(r'/(?P<width>\d+)x(?P<height>\d+)/', video_url)
	37	if m:
	38	a_format.update({
	39	'width': int(m.group('width')),
	40	'height': int(m.group('height')),
	41	})
	42
445d72b8 YCH	43
445d72b8 YCH	44	class TwitterCardIE(TwitterBaseIE):
014e8803	45	IE_NAME = 'twitter:card'
748a462f	46	_VALID_URL = r'https?://(?:www\.)?twitter\.com/i/(?:cards/tfw/v1\|videos(?:/tweet)?)/(?P<id>\d+)'
c3dea3f8	47	_TESTS = [
	48	{
	49	'url': 'https://twitter.com/i/cards/tfw/v1/560070183650213889',
acb6e97e	50	# MD5 checksums are different in different places
c3dea3f8	51	'info_dict': {
	52	'id': '560070183650213889',
	53	'ext': 'mp4',
0ae937a7	54	'title': 'Twitter Card',
ec85ded8	55	'thumbnail': r're:^https?://.*\.jpg$',
c3dea3f8	56	'duration': 30.033,
3615bfe1 YCH	57	},
3615bfe1 YCH	58	'skip': 'Video gone',
23e7cba8	59	},
c3dea3f8	60	{
	61	'url': 'https://twitter.com/i/cards/tfw/v1/623160978427936768',
	62	'md5': '7ee2a553b63d1bccba97fbed97d9e1c8',
	63	'info_dict': {
	64	'id': '623160978427936768',
	65	'ext': 'mp4',
0ae937a7	66	'title': 'Twitter Card',
ec85ded8	67	'thumbnail': r're:^https?://.*\.jpg',
c3dea3f8	68	'duration': 80.155,
c3dea3f8	69	},
3615bfe1	70	'skip': 'Video gone',
4a7b7903 YCH	71	},
	72	{
	73	'url': 'https://twitter.com/i/cards/tfw/v1/654001591733886977',
f0bc5a86	74	'md5': 'b6d9683dd3f48e340ded81c0e917ad46',
4a7b7903 YCH	75	'info_dict': {
	76	'id': 'dq4Oj5quskI',
	77	'ext': 'mp4',
	78	'title': 'Ubuntu 11.10 Overview',
f0bc5a86	79	'description': 'md5:a831e97fa384863d6e26ce48d1c43376',
4a7b7903 YCH	80	'upload_date': '20111013',
	81	'uploader': 'OMG! Ubuntu!',
	82	'uploader_id': 'omgubuntu',
	83	},
31752f76	84	'add_ie': ['Youtube'],
5f1b2aea YCH	85	},
	86	{
	87	'url': 'https://twitter.com/i/cards/tfw/v1/665289828897005568',
3615bfe1	88	'md5': '6dabeaca9e68cbb71c99c322a4b42a11',
5f1b2aea YCH	89	'info_dict': {
	90	'id': 'iBb2x00UVlv',
	91	'ext': 'mp4',
	92	'upload_date': '20151113',
	93	'uploader_id': '1189339351084113920',
acb6e97e YCH	94	'uploader': 'ArsenalTerje',
acb6e97e YCH	95	'title': 'Vine by ArsenalTerje',
e8f20ffa	96	'timestamp': 1447451307,
5f1b2aea YCH	97	},
5f1b2aea YCH	98	'add_ie': ['Vine'],
0ae937a7 YCH	99	}, {
0ae937a7 YCH	100	'url': 'https://twitter.com/i/videos/tweet/705235433198714880',
3615bfe1	101	'md5': '884812a2adc8aaf6fe52b15ccbfa3b88',
0ae937a7 YCH	102	'info_dict': {
	103	'id': '705235433198714880',
	104	'ext': 'mp4',
	105	'title': 'Twitter web player',
3615bfe1	106	'thumbnail': r're:^https?://.*',
0ae937a7	107	},
748a462f S	108	}, {
	109	'url': 'https://twitter.com/i/videos/752274308186120192',
	110	'only_matching': True,
0ae937a7	111	},
c3dea3f8	112	]
23e7cba8	113
2edfd745 YCH	114	def _parse_media_info(self, media_info, video_id):
	115	formats = []
	116	for media_variant in media_info.get('variants', []):
	117	media_url = media_variant['url']
	118	if media_url.endswith('.m3u8'):
	119	formats.extend(self._extract_m3u8_formats(media_url, video_id, ext='mp4', m3u8_id='hls'))
	120	elif media_url.endswith('.mpd'):
	121	formats.extend(self._extract_mpd_formats(media_url, video_id, mpd_id='dash'))
	122	else:
	123	vbr = int_or_none(dict_get(media_variant, ('bitRate', 'bitrate')), scale=1000)
	124	a_format = {
	125	'url': media_url,
	126	'format_id': 'http-%d' % vbr if vbr else 'http',
	127	'vbr': vbr,
	128	}
	129	# Reported bitRate may be zero
	130	if not a_format['vbr']:
	131	del a_format['vbr']
	132
	133	self._search_dimensions_in_video_url(a_format, media_url)
	134
	135	formats.append(a_format)
	136	return formats
	137
	138	def _extract_mobile_formats(self, username, video_id):
	139	webpage = self._download_webpage(
	140	'https://mobile.twitter.com/%s/status/%s' % (username, video_id),
	141	video_id, 'Downloading mobile webpage',
	142	headers={
	143	# A recent mobile UA is necessary for `gt` cookie
	144	'User-Agent': 'Mozilla/5.0 (Android 6.0.1; Mobile; rv:54.0) Gecko/54.0 Firefox/54.0',
	145	})
	146	main_script_url = self._html_search_regex(
	147	r'<script[^>]+src="([^"]+main\.[^"]+)"', webpage, 'main script URL')
	148	main_script = self._download_webpage(
	149	main_script_url, video_id, 'Downloading main script')
	150	bearer_token = self._search_regex(
	151	r'BEARER_TOKEN\s:\s"([^"]+)"',
	152	main_script, 'bearer token')
	153	guest_token = self._search_regex(
	154	r'document\.cookie\s=\sdecodeURIComponent\("gt=(\d+)',
	155	webpage, 'guest token')
	156	api_data = self._download_json(
	157	'https://api.twitter.com/2/timeline/conversation/%s.json' % video_id,
	158	video_id, 'Downloading mobile API data',
	159	headers={
	160	'Authorization': 'Bearer ' + bearer_token,
	161	'x-guest-token': guest_token,
	162	})
	163	media_info = try_get(api_data, lambda o: o['globalObjects']['tweets'][video_id]
	164	['extended_entities']['media'][0]['video_info']) or {}
	165	return self._parse_media_info(media_info, video_id)
	166
23e7cba8 S	167	def _real_extract(self, url):
	168	video_id = self._match_id(url)
	169
23e7cba8 S	170	config = None
23e7cba8 S	171	formats = []
b8972bd6 YCH	172	duration = None
b8972bd6 YCH	173
c8398a9b YCH	174	webpage = self._download_webpage(url, video_id)
	175
	176	iframe_url = self._html_search_regex(
	177	r'<iframe[^>]+src="((?:https?:)?//(?:www.youtube.com/embed/[^"]+\|(?:www\.)?vine\.co/v/\w+/card))"',
	178	webpage, 'video iframe', default=None)
	179	if iframe_url:
	180	return self.url_result(iframe_url)
	181
	182	config = self._parse_json(self._html_search_regex(
f0bc5a86 YCH	183	r'data-(?:player-)?config="([^"]+)"', webpage,
f0bc5a86 YCH	184	'data player config', default='{}'),
c8398a9b YCH	185	video_id)
c8398a9b YCH	186
395fd4b0 YCH	187	if config.get('source_type') == 'vine':
	188	return self.url_result(config['player_url'], 'Vine')
	189
f0bc5a86 YCH	190	periscope_url = PeriscopeIE._extract_url(webpage)
	191	if periscope_url:
	192	return self.url_result(periscope_url, PeriscopeIE.ie_key())
	193
e68d3a01	194	video_url = config.get('video_url') or config.get('playlist', [{}])[0].get('source')
c8398a9b	195
e68d3a01	196	if video_url:
c6308b31 YCH	197	if determine_ext(video_url) == 'm3u8':
	198	formats.extend(self._extract_m3u8_formats(video_url, video_id, ext='mp4', m3u8_id='hls'))
	199	else:
	200	f = {
	201	'url': video_url,
	202	}
c8398a9b	203
2edfd745	204	self._search_dimensions_in_video_url(f, video_url)
845817aa	205
c6308b31	206	formats.append(f)
c8398a9b YCH	207
	208	vmap_url = config.get('vmapUrl') or config.get('vmap_url')
	209	if vmap_url:
9be31e77 YCH	210	formats.extend(
9be31e77 YCH	211	self._extract_formats_from_vmap_url(vmap_url, video_id))
c8398a9b	212
03879ff0 YCH	213	media_info = None
	214
	215	for entity in config.get('status', {}).get('entities', []):
	216	if 'mediaInfo' in entity:
	217	media_info = entity['mediaInfo']
	218
c8398a9b	219	if media_info:
2edfd745	220	formats.extend(self._parse_media_info(media_info, video_id))
c8398a9b	221	duration = float_or_none(media_info.get('duration', {}).get('nanos'), scale=1e9)
b8972bd6	222
2edfd745 YCH	223	username = config.get('user', {}).get('screen_name')
	224	if username:
	225	formats.extend(self._extract_mobile_formats(username, video_id))
	226
	227	self._remove_duplicate_formats(formats)
23e7cba8 S	228	self._sort_formats(formats)
23e7cba8 S	229
0ae937a7 YCH	230	title = self._search_regex(r'<title>([^<]+)</title>', webpage, 'title')
0ae937a7 YCH	231	thumbnail = config.get('posterImageUrl') or config.get('image_src')
b8972bd6	232	duration = float_or_none(config.get('duration')) or duration
23e7cba8 S	233
	234	return {
	235	'id': video_id,
0ae937a7	236	'title': title,
23e7cba8 S	237	'thumbnail': thumbnail,
	238	'duration': duration,
	239	'formats': formats,
	240	}
f57f84f6	241
f57f84f6	242
e04edad6	243	class TwitterIE(InfoExtractor):
014e8803	244	IE_NAME = 'twitter'
575036b4 YCH	245	_VALID_URL = r'https?://(?:www\.\|m\.\|mobile\.)?twitter\.com/(?P<user_id>[^/]+)/status/(?P<id>\d+)'
575036b4 YCH	246	_TEMPLATE_URL = 'https://twitter.com/%s/status/%s'
f57f84f6	247
cf5881fc	248	_TESTS = [{
48aae2d2	249	'url': 'https://twitter.com/freethenipple/status/643211948184596480',
f57f84f6	250	'info_dict': {
48aae2d2	251	'id': '643211948184596480',
f57f84f6	252	'ext': 'mp4',
575036b4	253	'title': 'FREE THE NIPPLE - FTN supporters on Hollywood Blvd today!',
ec85ded8	254	'thumbnail': r're:^https?://.*\.jpg',
48aae2d2 YCH	255	'description': 'FREE THE NIPPLE on Twitter: "FTN supporters on Hollywood Blvd today! http://t.co/c7jHH749xJ"',
	256	'uploader': 'FREE THE NIPPLE',
	257	'uploader_id': 'freethenipple',
f57f84f6	258	},
b8972bd6 YCH	259	'params': {
	260	'skip_download': True, # requires ffmpeg
	261	},
cf5881fc YCH	262	}, {
	263	'url': 'https://twitter.com/giphz/status/657991469417025536/photo/1',
	264	'md5': 'f36dcd5fb92bf7057f155e7d927eeb42',
	265	'info_dict': {
	266	'id': '657991469417025536',
	267	'ext': 'mp4',
	268	'title': 'Gifs - tu vai cai tu vai cai tu nao eh capaz disso tu vai cai',
	269	'description': 'Gifs on Twitter: "tu vai cai tu vai cai tu nao eh capaz disso tu vai cai https://t.co/tM46VHFlO5"',
ec85ded8	270	'thumbnail': r're:^https?://.*\.png',
cf5881fc YCH	271	'uploader': 'Gifs',
	272	'uploader_id': 'giphz',
	273	},
7efc1c2b	274	'expected_warnings': ['height', 'width'],
fc0a45fa	275	'skip': 'Account suspended',
b703ebee JMF	276	}, {
	277	'url': 'https://twitter.com/starwars/status/665052190608723968',
	278	'md5': '39b7199856dee6cd4432e72c74bc69d4',
	279	'info_dict': {
	280	'id': '665052190608723968',
	281	'ext': 'mp4',
	282	'title': 'Star Wars - A new beginning is coming December 18. Watch the official 60 second #TV spot for #StarWars: #TheForceAwakens.',
	283	'description': 'Star Wars on Twitter: "A new beginning is coming December 18. Watch the official 60 second #TV spot for #StarWars: #TheForceAwakens."',
	284	'uploader_id': 'starwars',
	285	'uploader': 'Star Wars',
	286	},
0ae937a7 YCH	287	}, {
	288	'url': 'https://twitter.com/BTNBrentYarina/status/705235433198714880',
	289	'info_dict': {
	290	'id': '705235433198714880',
	291	'ext': 'mp4',
	292	'title': 'Brent Yarina - Khalil Iverson\'s missed highlight dunk. And made highlight dunk. In one highlight.',
	293	'description': 'Brent Yarina on Twitter: "Khalil Iverson\'s missed highlight dunk. And made highlight dunk. In one highlight."',
	294	'uploader_id': 'BTNBrentYarina',
	295	'uploader': 'Brent Yarina',
	296	},
	297	'params': {
	298	# The same video as https://twitter.com/i/videos/tweet/705235433198714880
	299	# Test case of TwitterCardIE
	300	'skip_download': True,
	301	},
03879ff0 YCH	302	}, {
	303	'url': 'https://twitter.com/jaydingeer/status/700207533655363584',
	304	'md5': '',
	305	'info_dict': {
	306	'id': '700207533655363584',
	307	'ext': 'mp4',
3615bfe1 YCH	308	'title': 'Donte - BEAT PROD: @suhmeduh #Damndaniel',
3615bfe1 YCH	309	'description': 'Donte on Twitter: "BEAT PROD: @suhmeduh https://t.co/HBrQ4AfpvZ #Damndaniel https://t.co/byBooq2ejZ"',
ec85ded8	310	'thumbnail': r're:^https?://.*\.jpg',
3615bfe1	311	'uploader': 'Donte',
03879ff0 YCH	312	'uploader_id': 'jaydingeer',
	313	},
	314	'params': {
	315	'skip_download': True, # requires ffmpeg
	316	},
395fd4b0 YCH	317	}, {
	318	'url': 'https://twitter.com/Filmdrunk/status/713801302971588609',
	319	'md5': '89a15ed345d13b86e9a5a5e051fa308a',
	320	'info_dict': {
	321	'id': 'MIOxnrUteUd',
	322	'ext': 'mp4',
3615bfe1 YCH	323	'title': 'FilmDrunk - Vine of the day',
	324	'description': 'FilmDrunk on Twitter: "Vine of the day https://t.co/xmTvRdqxWf"',
	325	'uploader': 'FilmDrunk',
	326	'uploader_id': 'Filmdrunk',
	327	'timestamp': 1402826626,
395fd4b0 YCH	328	'upload_date': '20140615',
	329	},
	330	'add_ie': ['Vine'],
36b7d9db YCH	331	}, {
36b7d9db YCH	332	'url': 'https://twitter.com/captainamerica/status/719944021058060289',
36b7d9db YCH	333	'info_dict': {
	334	'id': '719944021058060289',
	335	'ext': 'mp4',
	336	'title': 'Captain America - @King0fNerd Are you sure you made the right choice? Find out in theaters.',
	337	'description': 'Captain America on Twitter: "@King0fNerd Are you sure you made the right choice? Find out in theaters. https://t.co/GpgYi9xMJI"',
	338	'uploader_id': 'captainamerica',
	339	'uploader': 'Captain America',
	340	},
c6308b31 YCH	341	'params': {
	342	'skip_download': True, # requires ffmpeg
	343	},
f0bc5a86 YCH	344	}, {
	345	'url': 'https://twitter.com/OPP_HSD/status/779210622571536384',
	346	'info_dict': {
	347	'id': '1zqKVVlkqLaKB',
	348	'ext': 'mp4',
7f176ac4 YCH	349	'title': 'Sgt Kerry Schmidt - LIVE on #Periscope: Road rage, mischief, assault, rollover and fire in one occurrence',
7f176ac4 YCH	350	'description': 'Sgt Kerry Schmidt on Twitter: "LIVE on #Periscope: Road rage, mischief, assault, rollover and fire in one occurrence https://t.co/EKrVgIXF3s"',
f0bc5a86 YCH	351	'upload_date': '20160923',
f0bc5a86 YCH	352	'uploader_id': 'OPP_HSD',
7f176ac4	353	'uploader': 'Sgt Kerry Schmidt',
f0bc5a86 YCH	354	'timestamp': 1474613214,
	355	},
	356	'add_ie': ['Periscope'],
2edfd745 YCH	357	}, {
	358	# has mp4 formats via mobile API
	359	'url': 'https://twitter.com/news_al3alm/status/852138619213144067',
	360	'info_dict': {
	361	'id': '852138619213144067',
	362	'ext': 'mp4',
	363	'title': 'عالم الأخبار - كلمة تاريخية بجلسة الجناسي التاريخية.. النائب خالد مؤنس العتيبي للمعارضين : اتقوا الله .. الظلم ظلمات يوم القيامة',
	364	'description': 'عالم الأخبار on Twitter: "كلمة تاريخية بجلسة الجناسي التاريخية.. النائب خالد مؤنس العتيبي للمعارضين : اتقوا الله .. الظلم ظلمات يوم القيامة https://t.co/xg6OhpyKfN"',
	365	'uploader': 'عالم الأخبار',
	366	'uploader_id': 'news_al3alm',
	367	},
	368	'params': {
	369	'format': 'best[format_id^=http-]',
	370	},
cf5881fc	371	}]
f57f84f6	372
f57f84f6	373	def _real_extract(self, url):
575036b4 YCH	374	mobj = re.match(self._VALID_URL, url)
	375	user_id = mobj.group('user_id')
	376	twid = mobj.group('id')
	377
fc0a45fa YCH	378	webpage, urlh = self._download_webpage_handle(
	379	self._TEMPLATE_URL % (user_id, twid), twid)
	380
	381	if 'twitter.com/account/suspended' in urlh.geturl():
	382	raise ExtractorError('Account suspended by Twitter.', expected=True)
575036b4 YCH	383
	384	username = remove_end(self._og_search_title(webpage), ' on Twitter')
	385
b703ebee	386	title = description = self._og_search_description(webpage).strip('').replace('\n', ' ').strip('“”')
575036b4 YCH	387
575036b4 YCH	388	# strip 'https -_t.co_BJYgOjSeGA' junk from filenames
b703ebee	389	title = re.sub(r'\s+(https?://[^ ]+)', '', title)
575036b4	390
cf5881fc	391	info = {
575036b4 YCH	392	'uploader_id': user_id,
575036b4 YCH	393	'uploader': username,
f57f84f6	394	'webpage_url': url,
b703ebee	395	'description': '%s on Twitter: "%s"' % (username, description),
f57f84f6	396	'title': username + ' - ' + title,
f57f84f6	397	}
cf5881fc	398
cf5881fc	399	mobj = re.search(r'''(?x)
7efc1c2b	400	<video[^>]+class="animated-gif"(?P<more_info>[^>]+)>\s*
cf5881fc YCH	401	<source[^>]+video-src="(?P<url>[^"]+)"
	402	''', webpage)
	403
	404	if mobj:
7efc1c2b YCH	405	more_info = mobj.group('more_info')
	406	height = int_or_none(self._search_regex(
	407	r'data-height="(\d+)"', more_info, 'height', fatal=False))
	408	width = int_or_none(self._search_regex(
	409	r'data-width="(\d+)"', more_info, 'width', fatal=False))
	410	thumbnail = self._search_regex(
	411	r'poster="([^"]+)"', more_info, 'poster', fatal=False)
cf5881fc YCH	412	info.update({
	413	'id': twid,
	414	'url': mobj.group('url'),
7efc1c2b YCH	415	'height': height,
	416	'width': width,
	417	'thumbnail': thumbnail,
cf5881fc YCH	418	})
	419	return info
	420
f0bc5a86	421	twitter_card_url = None
0ae937a7	422	if 'class="PlayableMedia' in webpage:
f0bc5a86 YCH	423	twitter_card_url = '%s//twitter.com/i/videos/tweet/%s' % (self.http_scheme(), twid)
	424	else:
	425	twitter_card_iframe_url = self._search_regex(
	426	r'data-full-card-iframe-url=([\'"])(?P<url>(?:(?!\1).)+)\1',
	427	webpage, 'Twitter card iframe URL', default=None, group='url')
	428	if twitter_card_iframe_url:
	429	twitter_card_url = compat_urlparse.urljoin(url, twitter_card_iframe_url)
	430
	431	if twitter_card_url:
0ae937a7 YCH	432	info.update({
	433	'_type': 'url_transparent',
	434	'ie_key': 'TwitterCard',
f0bc5a86	435	'url': twitter_card_url,
0ae937a7	436	})
0ae937a7 YCH	437	return info
0ae937a7 YCH	438
132e3b74	439	raise ExtractorError('There\'s no video in this tweet.')
445d72b8 YCH	440
	441
	442	class TwitterAmplifyIE(TwitterBaseIE):
	443	IE_NAME = 'twitter:amplify'
25042f73	444	_VALID_URL = r'https?://amp\.twimg\.com/v/(?P<id>[0-9a-f\-]{36})'
445d72b8 YCH	445
	446	_TEST = {
	447	'url': 'https://amp.twimg.com/v/0ba0c3c7-0af3-4c0a-bed5-7efd1ffa2951',
	448	'md5': '7df102d0b9fd7066b86f3159f8e81bf6',
	449	'info_dict': {
	450	'id': '0ba0c3c7-0af3-4c0a-bed5-7efd1ffa2951',
	451	'ext': 'mp4',
	452	'title': 'Twitter Video',
bdbf4ba4	453	'thumbnail': 're:^https?://.*',
445d72b8 YCH	454	},
	455	}
	456
	457	def _real_extract(self, url):
	458	video_id = self._match_id(url)
	459	webpage = self._download_webpage(url, video_id)
	460
	461	vmap_url = self._html_search_meta(
	462	'twitter:amplify:vmap', webpage, 'vmap url')
9be31e77	463	formats = self._extract_formats_from_vmap_url(vmap_url, video_id)
445d72b8	464
bdbf4ba4 YCH	465	thumbnails = []
	466	thumbnail = self._html_search_meta(
	467	'twitter:image:src', webpage, 'thumbnail', fatal=False)
	468
	469	def _find_dimension(target):
	470	w = int_or_none(self._html_search_meta(
	471	'twitter:%s:width' % target, webpage, fatal=False))
	472	h = int_or_none(self._html_search_meta(
	473	'twitter:%s:height' % target, webpage, fatal=False))
	474	return w, h
	475
	476	if thumbnail:
	477	thumbnail_w, thumbnail_h = _find_dimension('image')
	478	thumbnails.append({
	479	'url': thumbnail,
	480	'width': thumbnail_w,
	481	'height': thumbnail_h,
	482	})
	483
	484	video_w, video_h = _find_dimension('player')
9be31e77	485	formats[0].update({
bdbf4ba4 YCH	486	'width': video_w,
bdbf4ba4 YCH	487	'height': video_h,
9be31e77	488	})
bdbf4ba4	489
445d72b8 YCH	490	return {
	491	'id': video_id,
	492	'title': 'Twitter Video',
bdbf4ba4 YCH	493	'formats': formats,
bdbf4ba4 YCH	494	'thumbnails': thumbnails,
445d72b8	495	}