+import json
import re
+import urllib.error
from .common import InfoExtractor
from .periscope import PeriscopeBaseIE, PeriscopeIE
from ..compat import (
- compat_HTTPError,
compat_parse_qs,
compat_urllib_parse_unquote,
compat_urllib_parse_urlparse,
format_field,
int_or_none,
make_archive_id,
+ remove_end,
str_or_none,
strip_or_none,
traverse_obj,
+ try_call,
try_get,
unified_timestamp,
update_url_query,
class TwitterBaseIE(InfoExtractor):
+ _NETRC_MACHINE = 'twitter'
_API_BASE = 'https://api.twitter.com/1.1/'
- _BASE_REGEX = r'https?://(?:(?:www|m(?:obile)?)\.)?twitter\.com/'
- _GUEST_TOKEN = None
+ _GRAPHQL_API_BASE = 'https://twitter.com/i/api/graphql/'
+ _BASE_REGEX = r'https?://(?:(?:www|m(?:obile)?)\.)?(?:twitter\.com|twitter3e4tixl4xyajtrzo62zg5vztmjuricljdp2c5kshju4avyoid\.onion)/'
+ _AUTH = {'Authorization': 'Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA'}
+ _flow_token = None
+
+ _LOGIN_INIT_DATA = json.dumps({
+ 'input_flow_data': {
+ 'flow_context': {
+ 'debug_overrides': {},
+ 'start_location': {
+ 'location': 'unknown'
+ }
+ }
+ },
+ 'subtask_versions': {
+ 'action_list': 2,
+ 'alert_dialog': 1,
+ 'app_download_cta': 1,
+ 'check_logged_in_account': 1,
+ 'choice_selection': 3,
+ 'contacts_live_sync_permission_prompt': 0,
+ 'cta': 7,
+ 'email_verification': 2,
+ 'end_flow': 1,
+ 'enter_date': 1,
+ 'enter_email': 2,
+ 'enter_password': 5,
+ 'enter_phone': 2,
+ 'enter_recaptcha': 1,
+ 'enter_text': 5,
+ 'enter_username': 2,
+ 'generic_urt': 3,
+ 'in_app_notification': 1,
+ 'interest_picker': 3,
+ 'js_instrumentation': 1,
+ 'menu_dialog': 1,
+ 'notifications_permission_prompt': 2,
+ 'open_account': 2,
+ 'open_home_timeline': 1,
+ 'open_link': 1,
+ 'phone_verification': 4,
+ 'privacy_options': 1,
+ 'security_key': 3,
+ 'select_avatar': 4,
+ 'select_banner': 2,
+ 'settings_list': 7,
+ 'show_code': 1,
+ 'sign_up': 2,
+ 'sign_up_review': 4,
+ 'tweet_selection_urt': 1,
+ 'update_users': 1,
+ 'upload_media': 1,
+ 'user_recommendations_list': 4,
+ 'user_recommendations_urt': 1,
+ 'wait_spinner': 3,
+ 'web_modal': 1
+ }
+ }, separators=(',', ':')).encode()
def _extract_variant_formats(self, variant, video_id):
variant_url = variant.get('url')
'height': int(m.group('height')),
})
- def _call_api(self, path, video_id, query={}):
- headers = {
- 'Authorization': 'Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA',
- }
- token = self._get_cookies(self._API_BASE).get('ct0')
- if token:
- headers['x-csrf-token'] = token.value
- if not self._GUEST_TOKEN:
- self._GUEST_TOKEN = self._download_json(
- self._API_BASE + 'guest/activate.json', video_id,
- 'Downloading guest token', data=b'',
- headers=headers)['guest_token']
- headers['x-guest-token'] = self._GUEST_TOKEN
- try:
- return self._download_json(
- self._API_BASE + path, video_id, headers=headers, query=query)
- except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
- raise ExtractorError(self._parse_json(
- e.cause.read().decode(),
- video_id)['errors'][0]['message'], expected=True)
- raise
+ @property
+ def is_logged_in(self):
+ return bool(self._get_cookies(self._API_BASE).get('auth_token'))
+
+ def _set_base_headers(self):
+ headers = self._AUTH.copy()
+ csrf_token = try_call(lambda: self._get_cookies(self._API_BASE)['ct0'].value)
+ if csrf_token:
+ headers['x-csrf-token'] = csrf_token
+ return headers
+
+ def _call_login_api(self, note, headers, query={}, data=None):
+ response = self._download_json(
+ f'{self._API_BASE}onboarding/task.json', None, note,
+ headers=headers, query=query, data=data, expected_status=400)
+ error = traverse_obj(response, ('errors', 0, 'message', {str}))
+ if error:
+ raise ExtractorError(f'Login failed, Twitter API says: {error}', expected=True)
+ elif traverse_obj(response, 'status') != 'success':
+ raise ExtractorError('Login was unsuccessful')
+
+ subtask = traverse_obj(
+ response, ('subtasks', ..., 'subtask_id', {str}), get_all=False)
+ if not subtask:
+ raise ExtractorError('Twitter API did not return next login subtask')
+
+ self._flow_token = response['flow_token']
+
+ return subtask
+
+ def _perform_login(self, username, password):
+ if self.is_logged_in:
+ return
+
+ webpage = self._download_webpage('https://twitter.com/', None, 'Downloading login page')
+ headers = self._set_base_headers()
+ guest_token = self._search_regex(
+ r'\.cookie\s*=\s*["\']gt=(\d+);', webpage, 'gt', default=None) or self._download_json(
+ f'{self._API_BASE}guest/activate.json', None, 'Downloading guest token',
+ data=b'', headers=headers)['guest_token']
+ headers.update({
+ 'content-type': 'application/json',
+ 'x-guest-token': guest_token,
+ 'x-twitter-client-language': 'en',
+ 'x-twitter-active-user': 'yes',
+ 'Referer': 'https://twitter.com/',
+ 'Origin': 'https://twitter.com',
+ })
+
+ def build_login_json(*subtask_inputs):
+ return json.dumps({
+ 'flow_token': self._flow_token,
+ 'subtask_inputs': subtask_inputs
+ }, separators=(',', ':')).encode()
+
+ def input_dict(subtask_id, text):
+ return {
+ 'subtask_id': subtask_id,
+ 'enter_text': {
+ 'text': text,
+ 'link': 'next_link'
+ }
+ }
+
+ next_subtask = self._call_login_api(
+ 'Downloading flow token', headers, query={'flow_name': 'login'}, data=self._LOGIN_INIT_DATA)
+
+ while not self.is_logged_in:
+ if next_subtask == 'LoginJsInstrumentationSubtask':
+ next_subtask = self._call_login_api(
+ 'Submitting JS instrumentation response', headers, data=build_login_json({
+ 'subtask_id': next_subtask,
+ 'js_instrumentation': {
+ 'response': '{}',
+ 'link': 'next_link'
+ }
+ }))
+
+ elif next_subtask == 'LoginEnterUserIdentifierSSO':
+ next_subtask = self._call_login_api(
+ 'Submitting username', headers, data=build_login_json({
+ 'subtask_id': next_subtask,
+ 'settings_list': {
+ 'setting_responses': [{
+ 'key': 'user_identifier',
+ 'response_data': {
+ 'text_data': {
+ 'result': username
+ }
+ }
+ }],
+ 'link': 'next_link'
+ }
+ }))
+
+ elif next_subtask == 'LoginEnterAlternateIdentifierSubtask':
+ next_subtask = self._call_login_api(
+ 'Submitting alternate identifier', headers,
+ data=build_login_json(input_dict(next_subtask, self._get_tfa_info(
+ 'one of username, phone number or email that was not used as --username'))))
+
+ elif next_subtask == 'LoginEnterPassword':
+ next_subtask = self._call_login_api(
+ 'Submitting password', headers, data=build_login_json({
+ 'subtask_id': next_subtask,
+ 'enter_password': {
+ 'password': password,
+ 'link': 'next_link'
+ }
+ }))
+
+ elif next_subtask == 'AccountDuplicationCheck':
+ next_subtask = self._call_login_api(
+ 'Submitting account duplication check', headers, data=build_login_json({
+ 'subtask_id': next_subtask,
+ 'check_logged_in_account': {
+ 'link': 'AccountDuplicationCheck_false'
+ }
+ }))
+
+ elif next_subtask == 'LoginTwoFactorAuthChallenge':
+ next_subtask = self._call_login_api(
+ 'Submitting 2FA token', headers, data=build_login_json(input_dict(
+ next_subtask, self._get_tfa_info('two-factor authentication token'))))
+
+ elif next_subtask == 'LoginAcid':
+ next_subtask = self._call_login_api(
+ 'Submitting confirmation code', headers, data=build_login_json(input_dict(
+ next_subtask, self._get_tfa_info('confirmation code sent to your email or phone'))))
+
+ elif next_subtask == 'LoginSuccessSubtask':
+ raise ExtractorError('Twitter API did not grant auth token cookie')
+
+ else:
+ raise ExtractorError(f'Unrecognized subtask ID "{next_subtask}"')
+
+ self.report_login()
+
+ def _call_api(self, path, video_id, query={}, graphql=False):
+ if not self.is_logged_in:
+ self.raise_login_required()
+
+ result = self._download_json(
+ (self._GRAPHQL_API_BASE if graphql else self._API_BASE) + path, video_id,
+ f'Downloading {"GraphQL" if graphql else "legacy API"} JSON', headers={
+ **self._set_base_headers(),
+ 'x-twitter-auth-type': 'OAuth2Session',
+ 'x-twitter-client-language': 'en',
+ 'x-twitter-active-user': 'yes',
+ }, query=query, expected_status={400, 401, 403, 404} if graphql else {403})
+
+ if result.get('errors'):
+ errors = ', '.join(set(traverse_obj(result, ('errors', ..., 'message', {str}))))
+ raise ExtractorError(
+ f'Error(s) while querying API: {errors or "Unknown error"}', expected=True)
+
+ return result
+
+ def _build_graphql_query(self, media_id):
+ raise NotImplementedError('Method must be implemented to support GraphQL')
+
+ def _call_graphql_api(self, endpoint, media_id):
+ data = self._build_graphql_query(media_id)
+ query = {key: json.dumps(value, separators=(',', ':')) for key, value in data.items()}
+ return traverse_obj(self._call_api(endpoint, media_id, query=query, graphql=True), 'data')
class TwitterCardIE(InfoExtractor):
'url': 'https://twitter.com/i/cards/tfw/v1/560070183650213889',
# MD5 checksums are different in different places
'info_dict': {
- 'id': '560070183650213889',
+ 'id': '560070131976392705',
'ext': 'mp4',
'title': "Twitter - You can now shoot, edit and share video on Twitter. Capture life's most moving moments from your perspective.",
'description': 'md5:18d3e24bb4f6e5007487dd546e53bd96',
'duration': 30.033,
'timestamp': 1422366112,
'upload_date': '20150127',
+ 'age_limit': 0,
+ 'comment_count': int,
+ 'tags': [],
+ 'repost_count': int,
+ 'like_count': int,
+ 'display_id': '560070183650213889',
+ 'uploader_url': 'https://twitter.com/Twitter',
},
},
{
'uploader_id': 'NASA',
'timestamp': 1437408129,
'upload_date': '20150720',
+ 'uploader_url': 'https://twitter.com/NASA',
+ 'age_limit': 0,
+ 'comment_count': int,
+ 'like_count': int,
+ 'repost_count': int,
+ 'tags': ['PlutoFlyby'],
},
+ 'params': {'format': '[protocol=https]'}
},
{
'url': 'https://twitter.com/i/cards/tfw/v1/654001591733886977',
'upload_date': '20111013',
'uploader': 'OMG! UBUNTU!',
'uploader_id': 'omgubuntu',
+ 'channel_url': 'https://www.youtube.com/channel/UCIiSwcm9xiFb3Y4wjzR41eQ',
+ 'channel_id': 'UCIiSwcm9xiFb3Y4wjzR41eQ',
+ 'channel_follower_count': int,
+ 'chapters': 'count:8',
+ 'uploader_url': 'http://www.youtube.com/user/omgubuntu',
+ 'duration': 138,
+ 'categories': ['Film & Animation'],
+ 'age_limit': 0,
+ 'comment_count': int,
+ 'availability': 'public',
+ 'like_count': int,
+ 'thumbnail': 'https://i.ytimg.com/vi/dq4Oj5quskI/maxresdefault.jpg',
+ 'view_count': int,
+ 'tags': 'count:12',
+ 'channel': 'OMG! UBUNTU!',
+ 'playable_in_embed': True,
},
'add_ie': ['Youtube'],
},
{
'url': 'https://twitter.com/i/cards/tfw/v1/665289828897005568',
- 'md5': '6dabeaca9e68cbb71c99c322a4b42a11',
'info_dict': {
'id': 'iBb2x00UVlv',
'ext': 'mp4',
'uploader': 'ArsenalTerje',
'title': 'Vine by ArsenalTerje',
'timestamp': 1447451307,
+ 'alt_title': 'Vine by ArsenalTerje',
+ 'comment_count': int,
+ 'like_count': int,
+ 'thumbnail': r're:^https?://[^?#]+\.jpg',
+ 'view_count': int,
+ 'repost_count': int,
},
'add_ie': ['Vine'],
- }, {
+ 'params': {'skip_download': 'm3u8'},
+ },
+ {
'url': 'https://twitter.com/i/videos/tweet/705235433198714880',
'md5': '884812a2adc8aaf6fe52b15ccbfa3b88',
'info_dict': {
'upload_date': '20160303',
},
'skip': 'This content is no longer available.',
- }, {
+ },
+ {
'url': 'https://twitter.com/i/videos/752274308186120192',
'only_matching': True,
},
class TwitterIE(TwitterBaseIE):
IE_NAME = 'twitter'
- _VALID_URL = TwitterBaseIE._BASE_REGEX + r'(?:(?:i/web|[^/]+)/status|statuses)/(?P<id>\d+)'
+ _VALID_URL = TwitterBaseIE._BASE_REGEX + r'(?:(?:i/web|[^/]+)/status|statuses)/(?P<id>\d+)(?:/(?:video|photo)/(?P<index>\d+))?'
_TESTS = [{
+ # comment_count, repost_count, view_count are only available with auth (applies to all tests)
'url': 'https://twitter.com/freethenipple/status/643211948184596480',
'info_dict': {
'id': '643211870443208704',
'duration': 12.922,
'timestamp': 1442188653,
'upload_date': '20150913',
- 'age_limit': 18,
'uploader_url': 'https://twitter.com/freethenipple',
- 'comment_count': int,
- 'repost_count': int,
'like_count': int,
'tags': [],
'age_limit': 18,
'id': '665052190608723968',
'display_id': '665052190608723968',
'ext': 'mp4',
- 'title': 'Star Wars - A new beginning is coming December 18. Watch the official 60 second #TV spot for #StarWars: #TheForceAwakens.',
+ 'title': r're:Star Wars.*A new beginning is coming December 18.*',
'description': 'A new beginning is coming December 18. Watch the official 60 second #TV spot for #StarWars: #TheForceAwakens. https://t.co/OkSqT2fjWJ',
'uploader_id': 'starwars',
- 'uploader': 'Star Wars',
+ 'uploader': r're:Star Wars.*',
'timestamp': 1447395772,
'upload_date': '20151113',
'uploader_url': 'https://twitter.com/starwars',
- 'comment_count': int,
- 'repost_count': int,
'like_count': int,
'tags': ['TV', 'StarWars', 'TheForceAwakens'],
'age_limit': 0,
# Test case of TwitterCardIE
'skip_download': True,
},
+ 'skip': 'Dead external link',
}, {
'url': 'https://twitter.com/jaydingeer/status/700207533655363584',
'info_dict': {
'timestamp': 1455777459,
'upload_date': '20160218',
'uploader_url': 'https://twitter.com/jaydingeer',
- 'comment_count': int,
- 'repost_count': int,
'like_count': int,
'tags': ['Damndaniel'],
'age_limit': 0,
'upload_date': '20160412',
'uploader_url': 'https://twitter.com/CaptainAmerica',
'thumbnail': r're:^https?://.*\.jpg',
- 'comment_count': int,
- 'repost_count': int,
'like_count': int,
'tags': [],
'age_limit': 0,
'timestamp': 1505803395,
'upload_date': '20170919',
'uploader_url': 'https://twitter.com/Prefet971',
- 'comment_count': int,
- 'repost_count': int,
'like_count': int,
'tags': ['Maria'],
'age_limit': 0,
'timestamp': 1527623489,
'upload_date': '20180529',
'uploader_url': 'https://twitter.com/LisPower1',
- 'comment_count': int,
- 'repost_count': int,
'like_count': int,
'tags': [],
'age_limit': 0,
'timestamp': 1548184644,
'upload_date': '20190122',
'uploader_url': 'https://twitter.com/Twitter',
- 'comment_count': int,
- 'repost_count': int,
'like_count': int,
'tags': [],
'age_limit': 0,
'view_count': int,
},
'add_ie': ['TwitterBroadcast'],
+ 'skip': 'Requires authentication',
}, {
# unified card
'url': 'https://twitter.com/BrooklynNets/status/1349794411333394432?s=20',
'timestamp': 1610651040,
'upload_date': '20210114',
'uploader_url': 'https://twitter.com/BrooklynNets',
- 'comment_count': int,
- 'repost_count': int,
'like_count': int,
'tags': [],
'age_limit': 0,
'id': '1577855447914409984',
'display_id': '1577855540407197696',
'ext': 'mp4',
- 'title': 'oshtru \U0001faac\U0001f47d - gm \u2728\ufe0f now I can post image and video. nice update.',
- 'description': 'gm \u2728\ufe0f now I can post image and video. nice update. https://t.co/cG7XgiINOm',
+ 'title': 'md5:9d198efb93557b8f8d5b78c480407214',
+ 'description': 'md5:b9c3699335447391d11753ab21c70a74',
'upload_date': '20221006',
- 'uploader': 'oshtru \U0001faac\U0001f47d',
+ 'uploader': 'oshtru',
'uploader_id': 'oshtru',
'uploader_url': 'https://twitter.com/oshtru',
'thumbnail': r're:^https?://.*\.jpg',
'duration': 30.03,
- 'timestamp': 1665025050.0,
- 'comment_count': int,
- 'repost_count': int,
+ 'timestamp': 1665025050,
'like_count': int,
'tags': [],
'age_limit': 0,
'url': 'https://twitter.com/UltimaShadowX/status/1577719286659006464',
'info_dict': {
'id': '1577719286659006464',
- 'title': 'Ultima | #\u0432\u029f\u043c - Test',
+ 'title': 'Ultima📛 | #вʟм - Test',
'description': 'Test https://t.co/Y3KEZD7Dad',
- 'uploader': 'Ultima | #\u0432\u029f\u043c',
+ 'uploader': 'Ultima📛 | #вʟм',
'uploader_id': 'UltimaShadowX',
'uploader_url': 'https://twitter.com/UltimaShadowX',
'upload_date': '20221005',
- 'timestamp': 1664992565.0,
- 'comment_count': int,
- 'repost_count': int,
+ 'timestamp': 1664992565,
'like_count': int,
'tags': [],
'age_limit': 0,
},
'playlist_count': 4,
'params': {'skip_download': True},
+ }, {
+ 'url': 'https://twitter.com/MesoMax919/status/1575560063510810624',
+ 'info_dict': {
+ 'id': '1575559336759263233',
+ 'display_id': '1575560063510810624',
+ 'ext': 'mp4',
+ 'title': 'md5:eec26382babd0f7c18f041db8ae1c9c9',
+ 'thumbnail': r're:^https?://.*\.jpg',
+ 'description': 'md5:95aea692fda36a12081b9629b02daa92',
+ 'uploader': 'Max Olson',
+ 'uploader_id': 'MesoMax919',
+ 'uploader_url': 'https://twitter.com/MesoMax919',
+ 'duration': 21.321,
+ 'timestamp': 1664477766,
+ 'upload_date': '20220929',
+ 'like_count': int,
+ 'tags': ['HurricaneIan'],
+ 'age_limit': 0,
+ },
+ }, {
+ # Adult content, fails if not logged in (GraphQL)
+ 'url': 'https://twitter.com/Rizdraws/status/1575199173472927762',
+ 'info_dict': {
+ 'id': '1575199163847000068',
+ 'display_id': '1575199173472927762',
+ 'ext': 'mp4',
+ 'title': str,
+ 'description': str,
+ 'uploader': str,
+ 'uploader_id': 'Rizdraws',
+ 'uploader_url': 'https://twitter.com/Rizdraws',
+ 'upload_date': '20220928',
+ 'timestamp': 1664391723,
+ 'thumbnail': r're:^https?://.+\.jpg',
+ 'like_count': int,
+ 'repost_count': int,
+ 'comment_count': int,
+ 'age_limit': 18,
+ 'tags': []
+ },
+ 'skip': 'Requires authentication',
+ }, {
+ # Single Vimeo video result without auth
+ 'url': 'https://twitter.com/Srirachachau/status/1395079556562706435',
+ 'info_dict': {
+ 'id': '551578322',
+ 'ext': 'mp4',
+ 'title': 'Dusty & The Mayor',
+ 'uploader': 'Michael Chau',
+ 'uploader_id': 'user29061007',
+ 'uploader_url': 'https://vimeo.com/user29061007',
+ 'duration': 478,
+ 'thumbnail': 'https://i.vimeocdn.com/video/1139658575-0dfdce6e9a2401fe09feb24bf0d14e6f24a53c12f447ff688ace61009ad4c1ba-d_1280',
+ },
+ }, {
+ # Playlist result only with auth
+ 'url': 'https://twitter.com/Srirachachau/status/1395079556562706435',
+ 'playlist_mincount': 2,
+ 'info_dict': {
+ 'id': '1395079556562706435',
+ 'title': str,
+ 'tags': [],
+ 'uploader': str,
+ 'like_count': int,
+ 'upload_date': '20210519',
+ 'age_limit': 0,
+ 'repost_count': int,
+ 'description': 'Here it is! Finished my gothic western cartoon. Pretty proud of it. It\'s got some goofs and lots of splashy over the top violence, something for everyone, hope you like it https://t.co/fOsG5glUnw',
+ 'uploader_id': 'Srirachachau',
+ 'comment_count': int,
+ 'uploader_url': 'https://twitter.com/Srirachachau',
+ 'timestamp': 1621447860,
+ },
+ 'skip': 'Requires authentication',
+ }, {
+ 'url': 'https://twitter.com/DavidToons_/status/1578353380363501568',
+ 'playlist_mincount': 2,
+ 'info_dict': {
+ 'id': '1578353380363501568',
+ 'title': str,
+ 'uploader_id': 'DavidToons_',
+ 'repost_count': int,
+ 'like_count': int,
+ 'uploader': str,
+ 'timestamp': 1665143744,
+ 'uploader_url': 'https://twitter.com/DavidToons_',
+ 'description': 'Chris sounds like Linda from Bob\'s Burgers, so as an animator: this had to be done. https://t.co/WgJauwIW1w',
+ 'tags': [],
+ 'comment_count': int,
+ 'upload_date': '20221007',
+ 'age_limit': 0,
+ },
+ 'skip': 'Requires authentication',
+ }, {
+ 'url': 'https://twitter.com/primevideouk/status/1578401165338976258',
+ 'playlist_count': 2,
+ 'info_dict': {
+ 'id': '1578401165338976258',
+ 'title': str,
+ 'description': 'md5:659a6b517a034b4cee5d795381a2dc41',
+ 'uploader': str,
+ 'uploader_id': 'primevideouk',
+ 'timestamp': 1665155137,
+ 'upload_date': '20221007',
+ 'age_limit': 0,
+ 'uploader_url': 'https://twitter.com/primevideouk',
+ 'like_count': int,
+ 'tags': ['TheRingsOfPower'],
+ },
+ }, {
+ # Twitter Spaces
+ 'url': 'https://twitter.com/MoniqueCamarra/status/1550101959377551360',
+ 'info_dict': {
+ 'id': '1lPJqmBeeNAJb',
+ 'ext': 'm4a',
+ 'title': 'EuroFile@6 Ukraine Up-date-Draghi Defenestration-the West',
+ 'uploader': r're:Monique Camarra.+?',
+ 'uploader_id': 'MoniqueCamarra',
+ 'live_status': 'was_live',
+ 'release_timestamp': 1658417414,
+ 'description': 'md5:4dc8e972f1d8b3c6580376fabb02a3ad',
+ 'timestamp': 1658407771,
+ 'release_date': '20220721',
+ 'upload_date': '20220721',
+ },
+ 'add_ie': ['TwitterSpaces'],
+ 'params': {'skip_download': 'm3u8'},
+ 'skip': 'Requires authentication',
+ }, {
+ # URL specifies video number but --yes-playlist
+ 'url': 'https://twitter.com/CTVJLaidlaw/status/1600649710662213632/video/1',
+ 'playlist_mincount': 2,
+ 'info_dict': {
+ 'id': '1600649710662213632',
+ 'title': 'md5:be05989b0722e114103ed3851a0ffae2',
+ 'timestamp': 1670459604.0,
+ 'description': 'md5:591c19ce66fadc2359725d5cd0d1052c',
+ 'uploader_id': 'CTVJLaidlaw',
+ 'tags': ['colorectalcancer', 'cancerjourney', 'imnotaquitter'],
+ 'upload_date': '20221208',
+ 'age_limit': 0,
+ 'uploader': 'Jocelyn Laidlaw',
+ 'uploader_url': 'https://twitter.com/CTVJLaidlaw',
+ 'like_count': int,
+ },
+ }, {
+ # URL specifies video number and --no-playlist
+ 'url': 'https://twitter.com/CTVJLaidlaw/status/1600649710662213632/video/2',
+ 'info_dict': {
+ 'id': '1600649511827013632',
+ 'ext': 'mp4',
+ 'title': 'md5:7662a0a27ce6faa3e5b160340f3cfab1',
+ 'thumbnail': r're:^https?://.+\.jpg',
+ 'timestamp': 1670459604.0,
+ 'uploader_id': 'CTVJLaidlaw',
+ 'uploader': 'Jocelyn Laidlaw',
+ 'tags': ['colorectalcancer', 'cancerjourney', 'imnotaquitter'],
+ 'duration': 102.226,
+ 'uploader_url': 'https://twitter.com/CTVJLaidlaw',
+ 'display_id': '1600649710662213632',
+ 'like_count': int,
+ 'description': 'md5:591c19ce66fadc2359725d5cd0d1052c',
+ 'upload_date': '20221208',
+ 'age_limit': 0,
+ },
+ 'params': {'noplaylist': True},
+ }, {
+ # id pointing to TweetWithVisibilityResults type entity which wraps the actual Tweet over
+ # note the id different between extraction and url
+ 'url': 'https://twitter.com/s2FAKER/status/1621117700482416640',
+ 'info_dict': {
+ 'id': '1621117577354424321',
+ 'display_id': '1621117700482416640',
+ 'ext': 'mp4',
+ 'title': '뽀 - 아 최우제 이동속도 봐',
+ 'description': '아 최우제 이동속도 봐 https://t.co/dxu2U5vXXB',
+ 'duration': 24.598,
+ 'uploader': '뽀',
+ 'uploader_id': 's2FAKER',
+ 'uploader_url': 'https://twitter.com/s2FAKER',
+ 'upload_date': '20230202',
+ 'timestamp': 1675339553.0,
+ 'thumbnail': r're:https?://pbs\.twimg\.com/.+',
+ 'age_limit': 18,
+ 'tags': [],
+ 'like_count': int,
+ },
+ }, {
+ 'url': 'https://twitter.com/hlo_again/status/1599108751385972737/video/2',
+ 'info_dict': {
+ 'id': '1599108643743473680',
+ 'display_id': '1599108751385972737',
+ 'ext': 'mp4',
+ 'title': '\u06ea - \U0001F48B',
+ 'uploader_url': 'https://twitter.com/hlo_again',
+ 'like_count': int,
+ 'uploader_id': 'hlo_again',
+ 'thumbnail': 'https://pbs.twimg.com/ext_tw_video_thumb/1599108643743473680/pu/img/UG3xjov4rgg5sbYM.jpg?name=orig',
+ 'duration': 9.531,
+ 'upload_date': '20221203',
+ 'age_limit': 0,
+ 'timestamp': 1670092210.0,
+ 'tags': [],
+ 'uploader': '\u06ea',
+ 'description': '\U0001F48B https://t.co/bTj9Qz7vQP',
+ },
+ 'params': {'noplaylist': True},
+ }, {
+ 'url': 'https://twitter.com/MunTheShinobi/status/1600009574919962625',
+ 'info_dict': {
+ 'id': '1600009362759733248',
+ 'display_id': '1600009574919962625',
+ 'ext': 'mp4',
+ 'uploader_url': 'https://twitter.com/MunTheShinobi',
+ 'description': 'This is a genius ad by Apple. \U0001f525\U0001f525\U0001f525\U0001f525\U0001f525 https://t.co/cNsA0MoOml',
+ 'thumbnail': 'https://pbs.twimg.com/ext_tw_video_thumb/1600009362759733248/pu/img/XVhFQivj75H_YxxV.jpg?name=orig',
+ 'age_limit': 0,
+ 'uploader': 'Mün The Shinobi',
+ 'upload_date': '20221206',
+ 'title': 'Mün The Shinobi - This is a genius ad by Apple. \U0001f525\U0001f525\U0001f525\U0001f525\U0001f525',
+ 'like_count': int,
+ 'tags': [],
+ 'uploader_id': 'MunTheShinobi',
+ 'duration': 139.987,
+ 'timestamp': 1670306984.0,
+ },
+ }, {
+ # url to retweet id
+ 'url': 'https://twitter.com/liberdalau/status/1623739803874349067',
+ 'info_dict': {
+ 'id': '1623274794488659969',
+ 'display_id': '1623739803874349067',
+ 'ext': 'mp4',
+ 'title': 'Johnny Bullets - Me after going viral to over 30million people: Whoopsie-daisy',
+ 'description': 'md5:224d62f54b0cdef8e33d4c56c41ac503',
+ 'uploader': 'Johnny Bullets',
+ 'uploader_id': 'Johnnybull3ts',
+ 'uploader_url': 'https://twitter.com/Johnnybull3ts',
+ 'age_limit': 0,
+ 'tags': [],
+ 'duration': 8.033,
+ 'timestamp': 1675853859.0,
+ 'upload_date': '20230208',
+ 'thumbnail': r're:https://pbs\.twimg\.com/ext_tw_video_thumb/.+',
+ 'like_count': int,
+ },
+ }, {
+ # onion route
+ 'url': 'https://twitter3e4tixl4xyajtrzo62zg5vztmjuricljdp2c5kshju4avyoid.onion/TwitterBlue/status/1484226494708662273',
+ 'only_matching': True,
}, {
# Twitch Clip Embed
'url': 'https://twitter.com/GunB1g/status/1163218564784017422',
'only_matching': True,
}]
+ def _graphql_to_legacy(self, data, twid):
+ result = traverse_obj(data, (
+ 'threaded_conversation_with_injections_v2', 'instructions', 0, 'entries',
+ lambda _, v: v['entryId'] == f'tweet-{twid}', 'content', 'itemContent',
+ 'tweet_results', 'result', ('tweet', None),
+ ), expected_type=dict, default={}, get_all=False)
+
+ if result.get('__typename') not in ('Tweet', 'TweetTombstone', None):
+ self.report_warning(f'Unknown typename: {result.get("__typename")}', twid, only_once=True)
+
+ if 'tombstone' in result:
+ cause = remove_end(traverse_obj(result, ('tombstone', 'text', 'text', {str})), '. Learn more')
+ raise ExtractorError(f'Twitter API says: {cause or "Unknown error"}', expected=True)
+
+ status = result.get('legacy', {})
+ status.update(traverse_obj(result, {
+ 'user': ('core', 'user_results', 'result', 'legacy'),
+ 'card': ('card', 'legacy'),
+ 'quoted_status': ('quoted_status_result', 'result', 'legacy'),
+ }, expected_type=dict, default={}))
+
+ # extra transformation is needed since result does not match legacy format
+ binding_values = {
+ binding_value.get('key'): binding_value.get('value')
+ for binding_value in traverse_obj(status, ('card', 'binding_values', ..., {dict}))
+ }
+ if binding_values:
+ status['card']['binding_values'] = binding_values
+
+ return status
+
+ def _build_graphql_query(self, media_id):
+ return {
+ 'variables': {
+ 'focalTweetId': media_id,
+ 'includePromotedContent': True,
+ 'with_rux_injections': False,
+ 'withBirdwatchNotes': True,
+ 'withCommunity': True,
+ 'withDownvotePerspective': False,
+ 'withQuickPromoteEligibilityTweetFields': True,
+ 'withReactionsMetadata': False,
+ 'withReactionsPerspective': False,
+ 'withSuperFollowsTweetFields': True,
+ 'withSuperFollowsUserFields': True,
+ 'withV2Timeline': True,
+ 'withVoice': True,
+ },
+ 'features': {
+ 'graphql_is_translatable_rweb_tweet_is_translatable_enabled': False,
+ 'interactive_text_enabled': True,
+ 'responsive_web_edit_tweet_api_enabled': True,
+ 'responsive_web_enhance_cards_enabled': True,
+ 'responsive_web_graphql_timeline_navigation_enabled': False,
+ 'responsive_web_text_conversations_enabled': False,
+ 'responsive_web_uc_gql_enabled': True,
+ 'standardized_nudges_misinfo': True,
+ 'tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled': False,
+ 'tweetypie_unmention_optimization_enabled': True,
+ 'unified_cards_ad_metadata_container_dynamic_card_content_query_enabled': True,
+ 'verified_phone_label_enabled': False,
+ 'vibe_api_enabled': True,
+ },
+ }
+
def _real_extract(self, url):
- twid = self._match_id(url)
- status = self._call_api(
- 'statuses/show/%s.json' % twid, twid, {
- 'cards_platform': 'Web-12',
- 'include_cards': 1,
- 'include_reply_count': 1,
- 'include_user_entities': 0,
- 'tweet_mode': 'extended',
- })
+ twid, selected_index = self._match_valid_url(url).group('id', 'index')
+ if not self.is_logged_in:
+ try:
+ status = self._download_json(
+ 'https://cdn.syndication.twimg.com/tweet-result', twid, 'Downloading syndication JSON',
+ headers={'User-Agent': 'Googlebot'}, query={'id': twid})
+ self.to_screen(f'Some metadata is missing without authentication. {self._login_hint()}')
+ except ExtractorError as e:
+ if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 404:
+ self.raise_login_required('Requested tweet may only be available when logged in')
+ raise
+ else:
+ status = self._graphql_to_legacy(
+ self._call_graphql_api('zZXycP0V6H7m-2r0mOnFcA/TweetDetail', twid), twid)
- title = description = status['full_text'].replace('\n', ' ')
+ title = description = traverse_obj(
+ status, (('full_text', 'text'), {lambda x: x.replace('\n', ' ')}), get_all=False) or ''
# strip 'https -_t.co_BJYgOjSeGA' junk from filenames
title = re.sub(r'\s+(https?://[^ ]+)', '', title)
user = status.get('user') or {}
uploader = user.get('name')
if uploader:
- title = '%s - %s' % (uploader, title)
+ title = f'{uploader} - {title}'
uploader_id = user.get('screen_name')
- tags = []
- for hashtag in (try_get(status, lambda x: x['entities']['hashtags'], list) or []):
- hashtag_text = hashtag.get('text')
- if not hashtag_text:
- continue
- tags.append(hashtag_text)
-
info = {
'id': twid,
'title': title,
'repost_count': int_or_none(status.get('retweet_count')),
'comment_count': int_or_none(status.get('reply_count')),
'age_limit': 18 if status.get('possibly_sensitive') else 0,
- 'tags': tags,
+ 'tags': traverse_obj(status, ('entities', 'hashtags', ..., 'text')),
}
def extract_from_video_info(media):
media_id = traverse_obj(media, 'id_str', 'id', expected_type=str_or_none)
+ if not media_id:
+ # workaround for non-authenticated responses
+ media_id = traverse_obj(media, (
+ 'video_info', 'variants', ..., 'url',
+ {lambda x: re.search(r'_video/(\d+)/', x)[1]}), get_all=False)
self.write_debug(f'Extracting from video info: {media_id}')
- video_info = media.get('video_info') or {}
formats = []
subtitles = {}
- for variant in video_info.get('variants', []):
+ for variant in traverse_obj(media, ('video_info', 'variants', ...)):
fmts, subs = self._extract_variant_formats(variant, twid)
subtitles = self._merge_subtitles(subtitles, subs)
formats.extend(fmts)
- self._sort_formats(formats, ('res', 'br', 'size', 'proto')) # The codec of http formats are unknown
thumbnails = []
media_url = media.get('media_url_https') or media.get('media_url')
add_thumbnail('orig', media.get('original_info') or {})
return {
- 'id': media_id,
+ 'id': media_id or twid,
'formats': formats,
'subtitles': subtitles,
'thumbnails': thumbnails,
- 'duration': float_or_none(video_info.get('duration_millis'), 1000),
+ 'view_count': traverse_obj(media, ('mediaStats', 'viewCount', {int_or_none})),
+ 'duration': float_or_none(traverse_obj(media, ('video_info', 'duration_millis')), 1000),
+ # The codec of http formats are unknown
+ '_format_sort_fields': ('res', 'br', 'size', 'proto'),
}
def extract_from_card_info(card):
card_name = card['name'].split(':')[-1]
if card_name == 'player':
- return {
+ yield {
'_type': 'url',
'url': get_binding_value('player_url'),
}
elif card_name == 'periscope_broadcast':
- return {
+ yield {
'_type': 'url',
'url': get_binding_value('url') or get_binding_value('player_url'),
'ie_key': PeriscopeIE.ie_key(),
}
elif card_name == 'broadcast':
- return {
+ yield {
'_type': 'url',
'url': get_binding_value('broadcast_url'),
'ie_key': TwitterBroadcastIE.ie_key(),
}
+ elif card_name == 'audiospace':
+ yield {
+ '_type': 'url',
+ 'url': f'https://twitter.com/i/spaces/{get_binding_value("id")}',
+ 'ie_key': TwitterSpacesIE.ie_key(),
+ }
elif card_name == 'summary':
- return {
+ yield {
'_type': 'url',
'url': get_binding_value('card_url'),
}
elif card_name == 'unified_card':
- media_entities = self._parse_json(get_binding_value('unified_card'), twid)['media_entities']
- media = traverse_obj(media_entities, ..., expected_type=dict, get_all=False)
- return extract_from_video_info(media)
+ unified_card = self._parse_json(get_binding_value('unified_card'), twid)
+ yield from map(extract_from_video_info, traverse_obj(
+ unified_card, ('media_entities', ...), expected_type=dict))
# amplify, promo_video_website, promo_video_convo, appplayer,
# video_direct_message, poll2choice_video, poll3choice_video,
# poll4choice_video, ...
vmap_url = get_binding_value('amplify_url_vmap') if is_amplify else get_binding_value('player_stream_url')
content_id = get_binding_value('%s_content_id' % (card_name if is_amplify else 'player'))
formats, subtitles = self._extract_formats_from_vmap_url(vmap_url, content_id or twid)
- self._sort_formats(formats)
thumbnails = []
for suffix in ('_small', '', '_large', '_x_large', '_original'):
'height': int_or_none(image.get('height')),
})
- return {
+ yield {
'formats': formats,
'subtitles': subtitles,
'thumbnails': thumbnails,
'content_duration_seconds')),
}
- media_path = ((None, 'quoted_status'), 'extended_entities', 'media', lambda _, m: m['type'] != 'photo')
- videos = map(extract_from_video_info, traverse_obj(status, media_path, expected_type=dict))
- entries = [{**info, **data, 'display_id': twid} for data in videos if data]
+ videos = traverse_obj(status, (
+ ('mediaDetails', ((None, 'quoted_status'), 'extended_entities', 'media')),
+ lambda _, m: m['type'] != 'photo', {dict}))
+
+ if self._yes_playlist(twid, selected_index, video_label='URL-specified video number'):
+ selected_entries = (*map(extract_from_video_info, videos), *extract_from_card_info(status.get('card')))
+ else:
+ desired_obj = traverse_obj(status, (
+ ('mediaDetails', ((None, 'quoted_status'), 'extended_entities', 'media')),
+ int(selected_index) - 1, {dict}), get_all=False)
+ if not desired_obj:
+ raise ExtractorError(f'Video #{selected_index} is unavailable', expected=True)
+ elif desired_obj.get('type') != 'video':
+ raise ExtractorError(f'Media #{selected_index} is not a video', expected=True)
+
+ # Restore original archive id and video index in title
+ for index, entry in enumerate(videos, 1):
+ if entry.get('id') != desired_obj.get('id'):
+ continue
+ if index == 1:
+ info['_old_archive_ids'] = [make_archive_id(self, twid)]
+ if len(videos) != 1:
+ info['title'] += f' #{index}'
+ break
- data = extract_from_card_info(status.get('card'))
- if data:
- entries.append({**info, **data, 'display_id': twid})
+ return {**info, **extract_from_video_info(desired_obj), 'display_id': twid}
+ entries = [{**info, **data, 'display_id': twid} for data in selected_entries]
if not entries:
expanded_url = traverse_obj(status, ('entities', 'urls', 0, 'expanded_url'), expected_type=url_or_none)
if not expanded_url or expanded_url == url:
- raise ExtractorError('No video could be found in this tweet', expected=True)
+ self.raise_no_formats('No video could be found in this tweet', expected=True)
+ return info
return self.url_result(expanded_url, display_id=twid, **info)
_TEST = {
'url': 'https://amp.twimg.com/v/0ba0c3c7-0af3-4c0a-bed5-7efd1ffa2951',
- 'md5': '7df102d0b9fd7066b86f3159f8e81bf6',
+ 'md5': 'fec25801d18a4557c5c9f33d2c379ffa',
'info_dict': {
'id': '0ba0c3c7-0af3-4c0a-bed5-7efd1ffa2951',
'ext': 'mp4',
'title': 'Twitter Video',
'thumbnail': 're:^https?://.*',
},
+ 'params': {'format': '[protocol=https]'},
}
def _real_extract(self, url):
vmap_url = self._html_search_meta(
'twitter:amplify:vmap', webpage, 'vmap url')
- formats = self._extract_formats_from_vmap_url(vmap_url, video_id)
+ formats, _ = self._extract_formats_from_vmap_url(vmap_url, video_id)
thumbnails = []
thumbnail = self._html_search_meta(
'title': 'Andrea May Sahouri - Periscope Broadcast',
'uploader': 'Andrea May Sahouri',
'uploader_id': '1PXEdBZWpGwKe',
+ 'thumbnail': r're:^https?://[^?#]+\.jpg\?token=',
+ 'view_count': int,
},
}
info = self._parse_broadcast_data(broadcast, broadcast_id)
media_key = broadcast['media_key']
source = self._call_api(
- 'live_video_stream/status/' + media_key, media_key)['source']
+ f'live_video_stream/status/{media_key}', media_key)['source']
m3u8_url = source.get('noRedirectPlaybackUrl') or source['location']
if '/live_video_stream/geoblocked/' in m3u8_url:
self.raise_geo_restricted()
return info
+class TwitterSpacesIE(TwitterBaseIE):
+ IE_NAME = 'twitter:spaces'
+ _VALID_URL = TwitterBaseIE._BASE_REGEX + r'i/spaces/(?P<id>[0-9a-zA-Z]{13})'
+
+ _TESTS = [{
+ 'url': 'https://twitter.com/i/spaces/1RDxlgyvNXzJL',
+ 'info_dict': {
+ 'id': '1RDxlgyvNXzJL',
+ 'ext': 'm4a',
+ 'title': 'King Carlo e la mossa Kansas City per fare il Grande Centro',
+ 'description': 'Twitter Space participated by annarita digiorgio, Signor Ernesto, Raffaello Colosimo, Simone M. Sepe',
+ 'uploader': r're:Lucio Di Gaetano.*?',
+ 'uploader_id': 'luciodigaetano',
+ 'live_status': 'was_live',
+ 'timestamp': 1659877956,
+ 'upload_date': '20220807',
+ 'release_timestamp': 1659904215,
+ 'release_date': '20220807',
+ },
+ 'params': {'skip_download': 'm3u8'},
+ }]
+
+ SPACE_STATUS = {
+ 'notstarted': 'is_upcoming',
+ 'ended': 'was_live',
+ 'running': 'is_live',
+ 'timedout': 'post_live',
+ }
+
+ def _build_graphql_query(self, space_id):
+ return {
+ 'variables': {
+ 'id': space_id,
+ 'isMetatagsQuery': True,
+ 'withDownvotePerspective': False,
+ 'withReactionsMetadata': False,
+ 'withReactionsPerspective': False,
+ 'withReplays': True,
+ 'withSuperFollowsUserFields': True,
+ 'withSuperFollowsTweetFields': True,
+ },
+ 'features': {
+ 'dont_mention_me_view_api_enabled': True,
+ 'interactive_text_enabled': True,
+ 'responsive_web_edit_tweet_api_enabled': True,
+ 'responsive_web_enhance_cards_enabled': True,
+ 'responsive_web_uc_gql_enabled': True,
+ 'spaces_2022_h2_clipping': True,
+ 'spaces_2022_h2_spaces_communities': False,
+ 'standardized_nudges_misinfo': True,
+ 'tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled': False,
+ 'vibe_api_enabled': True,
+ },
+ }
+
+ def _real_extract(self, url):
+ space_id = self._match_id(url)
+ space_data = self._call_graphql_api('HPEisOmj1epUNLCWTYhUWw/AudioSpaceById', space_id)['audioSpace']
+ if not space_data:
+ raise ExtractorError('Twitter Space not found', expected=True)
+
+ metadata = space_data['metadata']
+ live_status = try_call(lambda: self.SPACE_STATUS[metadata['state'].lower()])
+ is_live = live_status == 'is_live'
+
+ formats = []
+ if live_status == 'is_upcoming':
+ self.raise_no_formats('Twitter Space not started yet', expected=True)
+ elif not is_live and not metadata.get('is_space_available_for_replay'):
+ self.raise_no_formats('Twitter Space ended and replay is disabled', expected=True)
+ elif metadata.get('media_key'):
+ source = traverse_obj(
+ self._call_api(f'live_video_stream/status/{metadata["media_key"]}', metadata['media_key']),
+ ('source', ('noRedirectPlaybackUrl', 'location'), {url_or_none}), get_all=False)
+ formats = self._extract_m3u8_formats(
+ source, metadata['media_key'], 'm4a', live=is_live, fatal=False,
+ headers={'Referer': 'https://twitter.com/'}) if source else []
+ for fmt in formats:
+ fmt.update({'vcodec': 'none', 'acodec': 'aac'})
+ if not is_live:
+ fmt['container'] = 'm4a_dash'
+
+ participants = ', '.join(traverse_obj(
+ space_data, ('participants', 'speakers', ..., 'display_name'))) or 'nobody yet'
+
+ if not formats and live_status == 'post_live':
+ self.raise_no_formats('Twitter Space ended but not downloadable yet', expected=True)
+
+ return {
+ 'id': space_id,
+ 'title': metadata.get('title'),
+ 'description': f'Twitter Space participated by {participants}',
+ 'uploader': traverse_obj(
+ metadata, ('creator_results', 'result', 'legacy', 'name')),
+ 'uploader_id': traverse_obj(
+ metadata, ('creator_results', 'result', 'legacy', 'screen_name')),
+ 'live_status': live_status,
+ 'release_timestamp': try_call(
+ lambda: int_or_none(metadata['scheduled_start'], scale=1000)),
+ 'timestamp': int_or_none(metadata.get('created_at'), scale=1000),
+ 'formats': formats,
+ }
+
+
class TwitterShortenerIE(TwitterBaseIE):
IE_NAME = 'twitter:shortener'
_VALID_URL = r'https?://t.co/(?P<id>[^?]+)|tco:(?P<eid>[^?]+)'
projects / yt-dlp.git / blobdiff