1 from .common
import InfoExtractor
2 from ..compat
import compat_urlparse
13 class NitterIE(InfoExtractor
):
14 # Taken from https://github.com/zedeus/nitter/wiki/Instances
16 NON_HTTP_INSTANCES
= (
17 '3nzoldnxplag42gqjs23xvghtzf6t6yzssrtytnntc6ppc7xxuoneoad.onion',
18 'nitter.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd.onion',
19 'nitter7bryz3jv7e3uekphigvmoyoem4al3fynerxkj22dmoxoq553qd.onion',
20 'npf37k3mtzwxreiw52ccs5ay4e6qt2fkcs2ndieurdyn2cuzzsfyfvid.onion',
21 'nitter.v6vgyqpa7yefkorazmg5d5fimstmvm2vtbirt6676mt7qmllrcnwycqd.onion',
22 'i23nv6w3juvzlw32xzoxcqzktegd4i4fu3nmnc2ewv4ggiu4ledwklad.onion',
23 '26oq3gioiwcmfojub37nz5gzbkdiqp7fue5kvye7d4txv4ny6fb4wwid.onion',
24 'vfaomgh4jxphpbdfizkm5gbtjahmei234giqj4facbwhrfjtcldauqad.onion',
25 'iwgu3cv7ywf3gssed5iqtavmrlszgsxazkmwwnt4h2kdait75thdyrqd.onion',
26 'erpnncl5nhyji3c32dcfmztujtl3xaddqb457jsbkulq24zqq7ifdgad.onion',
27 'ckzuw5misyahmg7j5t5xwwuj3bwy62jfolxyux4brfflramzsvvd3syd.onion',
28 'jebqj47jgxleaiosfcxfibx2xdahjettuydlxbg64azd4khsxv6kawid.onion',
29 'nttr2iupbb6fazdpr2rgbooon2tzbbsvvkagkgkwohhodjzj43stxhad.onion',
30 'nitraeju2mipeziu2wtcrqsxg7h62v5y4eqgwi75uprynkj74gevvuqd.onion',
31 'nitter.lqs5fjmajyp7rvp4qvyubwofzi6d4imua7vs237rkc4m5qogitqwrgyd.onion',
32 'ibsboeui2im5o7dxnik3s5yghufumgy5abevtij5nbizequfpu4qi4ad.onion',
33 'ec5nvbycpfa5k6ro77blxgkyrzbkv7uy6r5cngcbkadtjj2733nm3uyd.onion',
36 'u6ikd6zndl3c4dsdq4mmujpntgeevdk5qzkfb57r4tnfeccrn2qa.b32.i2p',
38 'nitterlgj3n5fgwesu3vxc5h67ruku33nqaoeoocae2mvlzhsu6k7fqd.onion',
43 'nitter.pussthecat.org',
44 'nitter.nixnet.services',
49 'nitter.domain.glass',
52 'nitter.actionsack.com',
53 'birdsite.xanny.family',
61 'twitter.grimneko.de',
62 'nitter.alefvanoon.xyz',
66 'twitter.mstdn.social',
69 'nitter.weiler.rocks',
70 'nitter.silkky.cloud',
71 'nitter.sethforprivacy.com',
73 'nitter.cutelab.space',
77 'fuckthesacklers.network',
79 'nitter.datatunnel.xyz',
80 'nitter.esmailelbob.xyz',
83 'nitter.winscloud.net',
84 'nitter.tiekoetter.com',
88 'nitter.mastodon.pro',
92 # not in the list anymore
101 # official, rate limited
104 'is-nitter.resolv.ee',
105 'lu-nitter.resolv.ee',
108 'nitter.cattube.org',
111 'nitter.himiko.cloud',
113 'nitter.mailstation.de',
114 'nitter.mastodont.cat',
115 'nitter.tedomum.net',
117 'nitter.weaponizedhumiliation.com',
118 'nitter.vxempire.xyz',
119 'tweet.lambda.dance',
122 INSTANCES
= NON_HTTP_INSTANCES
+ HTTP_INSTANCES
+ DEAD_INSTANCES
124 _INSTANCES_RE
= f
'(?:{"|".join(map(re.escape, INSTANCES))})'
125 _VALID_URL
= fr
'https?://{_INSTANCES_RE}/(?P<uploader_id>.+)/status/(?P<id>[0-9]+)(#.)?'
126 current_instance
= random
.choice(HTTP_INSTANCES
)
130 # GIF (wrapped in mp4)
131 'url': f
'https://{current_instance}/firefox/status/1314279897502629888#m',
133 'id': '1314279897502629888',
135 'title': 'md5:7890a9277da4639ab624dd899424c5d8',
136 'description': 'md5:5fea96a4d3716c350f8b95b21b3111fe',
137 'thumbnail': r
're:^https?://.*\.jpg$',
138 'uploader': 'Firefox 🔥',
139 'uploader_id': 'firefox',
140 'uploader_url': f
'https://{current_instance}/firefox',
141 'upload_date': '20201008',
142 'timestamp': 1602183720,
145 'comment_count': int,
148 'url': f
'https://{current_instance}/Le___Doc/status/1299715685392756737#m',
150 'id': '1299715685392756737',
152 'title': 're:^.* - "Je ne prédis jamais rien"\nD Raoult, Août 2020...',
153 'description': '"Je ne prédis jamais rien"\nD Raoult, Août 2020...',
154 'thumbnail': r
're:^https?://.*\.jpg$',
155 'uploader': 're:^Le *Doc',
156 'uploader_id': 'Le___Doc',
157 'uploader_url': f
'https://{current_instance}/Le___Doc',
158 'upload_date': '20200829',
159 'timestamp': 1598711340,
163 'comment_count': int,
165 }, { # video embed in a "Streaming Political Ads" box
166 'url': f
'https://{current_instance}/mozilla/status/1321147074491092994#m',
168 'id': '1321147074491092994',
170 'title': 'md5:8290664aabb43b9189145c008386bf12',
171 'description': 'md5:9cf2762d49674bc416a191a689fb2aaa',
172 'thumbnail': r
're:^https?://.*\.jpg$',
173 'uploader': 'Mozilla',
174 'uploader_id': 'mozilla',
175 'uploader_url': f
'https://{current_instance}/mozilla',
176 'upload_date': '20201027',
177 'timestamp': 1603820940,
181 'comment_count': int,
183 'expected_warnings': ['Ignoring subtitle tracks found in the HLS manifest'],
184 }, { # not the first tweet but main-tweet
185 'url': f
'https://{current_instance}/firefox/status/1354848277481414657#m',
187 'id': '1354848277481414657',
189 'title': 'md5:bef647f03bd1c6b15b687ea70dfc9700',
190 'description': 'md5:5efba25e2f9dac85ebcd21160cb4341f',
191 'thumbnail': r
're:^https?://.*\.jpg$',
192 'uploader': 'Firefox 🔥',
193 'uploader_id': 'firefox',
194 'uploader_url': f
'https://{current_instance}/firefox',
195 'upload_date': '20210128',
196 'timestamp': 1611855960,
200 'comment_count': int,
205 def _real_extract(self
, url
):
206 video_id
, uploader_id
= self
._match
_valid
_url
(url
).group('id', 'uploader_id')
207 parsed_url
= compat_urlparse
.urlparse(url
)
208 base_url
= f
'{parsed_url.scheme}://{parsed_url.netloc}'
210 self
._set
_cookie
(parsed_url
.netloc
, 'hlsPlayback', 'on')
211 full_webpage
= webpage
= self
._download
_webpage
(url
, video_id
)
213 main_tweet_start
= full_webpage
.find('class="main-tweet"')
214 if main_tweet_start
> 0:
215 webpage
= full_webpage
[main_tweet_start
:]
217 video_url
= '%s%s' % (base_url
, self
._html
_search
_regex
(
218 r
'(?:<video[^>]+data-url|<source[^>]+src)="([^"]+)"', webpage
, 'video url'))
219 ext
= determine_ext(video_url
)
221 if ext
== 'unknown_video':
222 formats
= self
._extract
_m
3u8_formats
(video_url
, video_id
, ext
='mp4')
229 title
= description
= self
._og
_search
_description
(full_webpage
) or self
._html
_search
_regex
(
230 r
'<div class="tweet-content[^>]+>([^<]+)</div>', webpage
, 'title', fatal
=False)
232 uploader_id
= self
._html
_search
_regex
(
233 r
'<a class="username"[^>]+title="@([^"]+)"', webpage
, 'uploader id', fatal
=False) or uploader_id
235 uploader
= self
._html
_search
_regex
(
236 r
'<a class="fullname"[^>]+title="([^"]+)"', webpage
, 'uploader name', fatal
=False)
238 title
= f
'{uploader} - {title}'
241 f
'{x[0]}_count': self
._html
_search
_regex
(
242 fr
'<span[^>]+class="icon-{x[1]}[^>]*></span>([^<]*)</div>',
243 webpage
, f
'{x[0]} count', fatal
=False)
244 for x
in (('view', 'play'), ('like', 'heart'), ('repost', 'retweet'), ('comment', 'comment'))
246 counts
= {field: 0 if count == '' else parse_count(count) for field, count in counts.items()}
249 self
._html
_search
_meta
('og:image', full_webpage
, 'thumbnail url')
250 or remove_end('%s%s' % (base_url
, self
._html
_search
_regex
(
251 r
'<video[^>]+poster="([^"]+)"', webpage
, 'thumbnail url', fatal
=False)), '%3Asmall'))
254 {'id': id, 'url': f'{thumbnail}
%3A{id}
'}
255 for id in ('thumb
', 'small
', 'large
', 'medium
', 'orig
')
258 date = self._html_search_regex(
259 r'<span
[^
>]+class="tweet-date"[^
>]*><a
[^
>]+title
="([^"]+)"',
260 webpage, 'upload date', default='').replace('·', '')
265 'description': description,
266 'uploader': uploader,
267 'timestamp': unified_timestamp(date),
268 'uploader_id': uploader_id,
269 'uploader_url': f'{base_url}/{uploader_id}',
271 'thumbnails': thumbnails,
272 'thumbnail': thumbnail,