-# coding: utf-8
-from __future__ import unicode_literals
+import random
+import re
+import urllib.parse
from .common import InfoExtractor
-from ..compat import compat_urlparse
from ..utils import (
+ determine_ext,
parse_count,
- unified_strdate,
- unified_timestamp,
remove_end,
- determine_ext,
+ unified_timestamp,
)
-import re
class NitterIE(InfoExtractor):
# Taken from https://github.com/zedeus/nitter/wiki/Instances
- INSTANCES = ('nitter.net',
- 'nitter.snopyta.org',
- 'nitter.42l.fr',
- 'nitter.nixnet.services',
- 'nitter.13ad.de',
- 'nitter.pussthecat.org',
- 'nitter.mastodont.cat',
- 'nitter.dark.fail',
- 'nitter.tedomum.net',
- 'nitter.cattube.org',
- 'nitter.fdn.fr',
- 'nitter.1d4.us',
- 'nitter.kavin.rocks',
- 'tweet.lambda.dance',
- 'nitter.cc',
- 'nitter.weaponizedhumiliation.com',
- '3nzoldnxplag42gqjs23xvghtzf6t6yzssrtytnntc6ppc7xxuoneoad.onion',
- 'nitter.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd.onion',
- 'nitterlgj3n5fgwesu3vxc5h67ruku33nqaoeoocae2mvlzhsu6k7fqd.onion')
-
- _INSTANCES_RE = '(?:' + '|'.join([re.escape(instance) for instance in INSTANCES]) + ')'
- _VALID_URL = r'https?://%(instance)s/(?P<uploader_id>.+)/status/(?P<id>[0-9]+)(#.)?' % {'instance': _INSTANCES_RE}
- current_instance = INSTANCES[0] # the test and official instance
+
+ NON_HTTP_INSTANCES = (
+ '3nzoldnxplag42gqjs23xvghtzf6t6yzssrtytnntc6ppc7xxuoneoad.onion',
+ 'nitter.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd.onion',
+ 'nitter7bryz3jv7e3uekphigvmoyoem4al3fynerxkj22dmoxoq553qd.onion',
+ 'npf37k3mtzwxreiw52ccs5ay4e6qt2fkcs2ndieurdyn2cuzzsfyfvid.onion',
+ 'nitter.v6vgyqpa7yefkorazmg5d5fimstmvm2vtbirt6676mt7qmllrcnwycqd.onion',
+ 'i23nv6w3juvzlw32xzoxcqzktegd4i4fu3nmnc2ewv4ggiu4ledwklad.onion',
+ '26oq3gioiwcmfojub37nz5gzbkdiqp7fue5kvye7d4txv4ny6fb4wwid.onion',
+ 'vfaomgh4jxphpbdfizkm5gbtjahmei234giqj4facbwhrfjtcldauqad.onion',
+ 'iwgu3cv7ywf3gssed5iqtavmrlszgsxazkmwwnt4h2kdait75thdyrqd.onion',
+ 'erpnncl5nhyji3c32dcfmztujtl3xaddqb457jsbkulq24zqq7ifdgad.onion',
+ 'ckzuw5misyahmg7j5t5xwwuj3bwy62jfolxyux4brfflramzsvvd3syd.onion',
+ 'jebqj47jgxleaiosfcxfibx2xdahjettuydlxbg64azd4khsxv6kawid.onion',
+ 'nttr2iupbb6fazdpr2rgbooon2tzbbsvvkagkgkwohhodjzj43stxhad.onion',
+ 'nitraeju2mipeziu2wtcrqsxg7h62v5y4eqgwi75uprynkj74gevvuqd.onion',
+ 'nitter.lqs5fjmajyp7rvp4qvyubwofzi6d4imua7vs237rkc4m5qogitqwrgyd.onion',
+ 'ibsboeui2im5o7dxnik3s5yghufumgy5abevtij5nbizequfpu4qi4ad.onion',
+ 'ec5nvbycpfa5k6ro77blxgkyrzbkv7uy6r5cngcbkadtjj2733nm3uyd.onion',
+
+ 'nitter.i2p',
+ 'u6ikd6zndl3c4dsdq4mmujpntgeevdk5qzkfb57r4tnfeccrn2qa.b32.i2p',
+
+ 'nitterlgj3n5fgwesu3vxc5h67ruku33nqaoeoocae2mvlzhsu6k7fqd.onion',
+ )
+
+ HTTP_INSTANCES = (
+ 'nitter.lacontrevoie.fr',
+ 'nitter.fdn.fr',
+ 'nitter.1d4.us',
+ 'nitter.kavin.rocks',
+ 'nitter.unixfox.eu',
+ 'nitter.domain.glass',
+ 'nitter.namazso.eu',
+ 'birdsite.xanny.family',
+ 'nitter.moomoo.me',
+ 'bird.trom.tf',
+ 'nitter.it',
+ 'twitter.censors.us',
+ 'nitter.grimneko.de',
+ 'twitter.076.ne.jp',
+ 'nitter.fly.dev',
+ 'notabird.site',
+ 'nitter.weiler.rocks',
+ 'nitter.sethforprivacy.com',
+ 'nitter.cutelab.space',
+ 'nitter.nl',
+ 'nitter.mint.lgbt',
+ 'nitter.bus-hit.me',
+ 'nitter.esmailelbob.xyz',
+ 'tw.artemislena.eu',
+ 'nitter.winscloud.net',
+ 'nitter.tiekoetter.com',
+ 'nitter.spaceint.fr',
+ 'nitter.privacy.com.de',
+ 'nitter.poast.org',
+ 'nitter.bird.froth.zone',
+ 'nitter.dcs0.hu',
+ 'twitter.dr460nf1r3.org',
+ 'nitter.garudalinux.org',
+ 'twitter.femboy.hu',
+ 'nitter.cz',
+ 'nitter.privacydev.net',
+ 'nitter.evil.site',
+ 'tweet.lambda.dance',
+ 'nitter.kylrth.com',
+ 'nitter.foss.wtf',
+ 'nitter.priv.pw',
+ 'nitter.tokhmi.xyz',
+ 'nitter.catalyst.sx',
+ 'unofficialbird.com',
+ 'nitter.projectsegfau.lt',
+ 'nitter.eu.projectsegfau.lt',
+ 'singapore.unofficialbird.com',
+ 'canada.unofficialbird.com',
+ 'india.unofficialbird.com',
+ 'nederland.unofficialbird.com',
+ 'uk.unofficialbird.com',
+ 'n.l5.ca',
+ 'nitter.slipfox.xyz',
+ 'nitter.soopy.moe',
+ 'nitter.qwik.space',
+ 'read.whatever.social',
+ 'nitter.rawbit.ninja',
+ 'nt.vern.cc',
+ 'ntr.odyssey346.dev',
+ 'nitter.ir',
+ 'nitter.privacytools.io',
+ 'nitter.sneed.network',
+ 'n.sneed.network',
+ 'nitter.manasiwibi.com',
+ 'nitter.smnz.de',
+ 'nitter.twei.space',
+ 'nitter.inpt.fr',
+ 'nitter.d420.de',
+ 'nitter.caioalonso.com',
+ 'nitter.at',
+ 'nitter.drivet.xyz',
+ 'nitter.pw',
+ 'nitter.nicfab.eu',
+ 'bird.habedieeh.re',
+ 'nitter.hostux.net',
+ 'nitter.adminforge.de',
+ 'nitter.platypush.tech',
+ 'nitter.mask.sh',
+ 'nitter.pufe.org',
+ 'nitter.us.projectsegfau.lt',
+ 'nitter.arcticfoxes.net',
+ 't.com.sb',
+ 'nitter.kling.gg',
+ 'nitter.ktachibana.party',
+ 'nitter.riverside.rocks',
+ 'nitter.girlboss.ceo',
+ 'nitter.lunar.icu',
+ 'twitter.moe.ngo',
+ 'nitter.freedit.eu',
+ 'ntr.frail.duckdns.org',
+ 'nitter.librenode.org',
+ 'n.opnxng.com',
+ 'nitter.plus.st',
+ )
+
+ DEAD_INSTANCES = (
+ # maintenance
+ 'nitter.ethibox.fr',
+
+ # official, rate limited
+ 'nitter.net',
+ # offline
+ 'is-nitter.resolv.ee',
+ 'lu-nitter.resolv.ee',
+ 'nitter.13ad.de',
+ 'nitter.40two.app',
+ 'nitter.cattube.org',
+ 'nitter.cc',
+ 'nitter.dark.fail',
+ 'nitter.himiko.cloud',
+ 'nitter.koyu.space',
+ 'nitter.mailstation.de',
+ 'nitter.mastodont.cat',
+ 'nitter.tedomum.net',
+ 'nitter.tokhmi.xyz',
+ 'nitter.weaponizedhumiliation.com',
+ 'nitter.vxempire.xyz',
+ 'tweet.lambda.dance',
+ 'nitter.ca',
+ 'nitter.42l.fr',
+ 'nitter.pussthecat.org',
+ 'nitter.nixnet.services',
+ 'nitter.eu',
+ 'nitter.actionsack.com',
+ 'nitter.hu',
+ 'twitr.gq',
+ 'nittereu.moomoo.me',
+ 'bird.from.tf',
+ 'twitter.grimneko.de',
+ 'nitter.alefvanoon.xyz',
+ 'n.hyperborea.cloud',
+ 'twitter.mstdn.social',
+ 'nitter.silkky.cloud',
+ 'nttr.stream',
+ 'fuckthesacklers.network',
+ 'nitter.govt.land',
+ 'nitter.datatunnel.xyz',
+ 'de.nttr.stream',
+ 'twtr.bch.bar',
+ 'nitter.exonip.de',
+ 'nitter.mastodon.pro',
+ 'nitter.notraxx.ch',
+ 'nitter.skrep.in',
+ 'nitter.snopyta.org',
+ )
+
+ INSTANCES = NON_HTTP_INSTANCES + HTTP_INSTANCES + DEAD_INSTANCES
+
+ _INSTANCES_RE = f'(?:{"|".join(map(re.escape, INSTANCES))})'
+ _VALID_URL = fr'https?://{_INSTANCES_RE}/(?P<uploader_id>.+)/status/(?P<id>[0-9]+)(#.)?'
+ current_instance = random.choice(HTTP_INSTANCES)
+
_TESTS = [
{
# GIF (wrapped in mp4)
- 'url': 'https://' + current_instance + '/firefox/status/1314279897502629888#m',
+ 'url': f'https://{current_instance}/firefox/status/1314279897502629888#m',
'info_dict': {
'id': '1314279897502629888',
'ext': 'mp4',
- 'title': 'Firefox 馃敟 - You know the old saying, if you see something say something. Now you actually can with the YouTube regrets extension. Report harmful YouTube recommendations so others can avoid watching them. 鉃★笍 https://mzl.la/3iFIiyg #UnfckTheInternet',
- 'description': 'You know the old saying, if you see something say something. Now you actually can with the YouTube regrets extension. Report harmful YouTube recommendations so others can avoid watching them. 鉃★笍 https://mzl.la/3iFIiyg #UnfckTheInternet',
+ 'title': 'md5:7890a9277da4639ab624dd899424c5d8',
+ 'description': 'md5:5fea96a4d3716c350f8b95b21b3111fe',
'thumbnail': r're:^https?://.*\.jpg$',
'uploader': 'Firefox 馃敟',
'uploader_id': 'firefox',
- 'uploader_url': 'https://' + current_instance + '/firefox',
+ 'uploader_url': f'https://{current_instance}/firefox',
'upload_date': '20201008',
'timestamp': 1602183720,
+ 'like_count': int,
+ 'repost_count': int,
+ 'comment_count': int,
},
}, { # normal video
- 'url': 'https://' + current_instance + '/Le___Doc/status/1299715685392756737#m',
+ 'url': f'https://{current_instance}/Le___Doc/status/1299715685392756737#m',
'info_dict': {
'id': '1299715685392756737',
'ext': 'mp4',
- 'title': 'Le Doc - "Je ne pr茅dis jamais rien" D Raoult, Ao没t 2020...',
- 'description': '"Je ne pr茅dis jamais rien" D Raoult, Ao没t 2020...',
+ 'title': 're:^.* - "Je ne pr茅dis jamais rien"\nD Raoult, Ao没t 2020...',
+ 'description': '"Je ne pr茅dis jamais rien"\nD Raoult, Ao没t 2020...',
'thumbnail': r're:^https?://.*\.jpg$',
- 'uploader': 'Le Doc',
+ 'uploader': 're:^Le *Doc',
'uploader_id': 'Le___Doc',
- 'uploader_url': 'https://' + current_instance + '/Le___Doc',
+ 'uploader_url': f'https://{current_instance}/Le___Doc',
'upload_date': '20200829',
- 'timestamp': 1598711341,
+ 'timestamp': 1598711340,
'view_count': int,
'like_count': int,
'repost_count': int,
'comment_count': int,
},
}, { # video embed in a "Streaming Political Ads" box
- 'url': 'https://' + current_instance + '/mozilla/status/1321147074491092994#m',
+ 'url': f'https://{current_instance}/mozilla/status/1321147074491092994#m',
'info_dict': {
'id': '1321147074491092994',
'ext': 'mp4',
- 'title': "Mozilla - Are you being targeted with weird, ominous or just plain annoying political ads while streaming your favorite shows? This isn't a real political ad, but if you're watching streaming TV in the U.S., chances are you've seen quite a few. Learn more 鉃★笍 https://mzl.la/StreamingAds",
- 'description': "Are you being targeted with weird, ominous or just plain annoying political ads while streaming your favorite shows? This isn't a real political ad, but if you're watching streaming TV in the U.S., chances are you've seen quite a few. Learn more 鉃★笍 https://mzl.la/StreamingAds",
+ 'title': 'md5:8290664aabb43b9189145c008386bf12',
+ 'description': 'md5:9cf2762d49674bc416a191a689fb2aaa',
'thumbnail': r're:^https?://.*\.jpg$',
'uploader': 'Mozilla',
'uploader_id': 'mozilla',
- 'uploader_url': 'https://' + current_instance + '/mozilla',
+ 'uploader_url': f'https://{current_instance}/mozilla',
'upload_date': '20201027',
- 'timestamp': 1603820982
+ 'timestamp': 1603820940,
+ 'view_count': int,
+ 'like_count': int,
+ 'repost_count': int,
+ 'comment_count': int,
},
+ 'expected_warnings': ['Ignoring subtitle tracks found in the HLS manifest'],
+ }, { # not the first tweet but main-tweet
+ 'url': f'https://{current_instance}/firefox/status/1354848277481414657#m',
+ 'info_dict': {
+ 'id': '1354848277481414657',
+ 'ext': 'mp4',
+ 'title': 'md5:bef647f03bd1c6b15b687ea70dfc9700',
+ 'description': 'md5:5efba25e2f9dac85ebcd21160cb4341f',
+ 'thumbnail': r're:^https?://.*\.jpg$',
+ 'uploader': 'Firefox 馃敟',
+ 'uploader_id': 'firefox',
+ 'uploader_url': f'https://{current_instance}/firefox',
+ 'upload_date': '20210128',
+ 'timestamp': 1611855960,
+ 'view_count': int,
+ 'like_count': int,
+ 'repost_count': int,
+ 'comment_count': int,
+ },
+ }, { # no OpenGraph title
+ 'url': f'https://{current_instance}/LocalBateman/status/1678455464038735895#m',
+ 'info_dict': {
+ 'id': '1678455464038735895',
+ 'ext': 'mp4',
+ 'title': 'Your Typical Local Man - Local man, what did Romanians ever do to you?',
+ 'description': 'Local man, what did Romanians ever do to you?',
+ 'thumbnail': r're:^https?://.*\.jpg$',
+ 'uploader': 'Your Typical Local Man',
+ 'uploader_id': 'LocalBateman',
+ 'uploader_url': f'https://{current_instance}/LocalBateman',
+ 'upload_date': '20230710',
+ 'timestamp': 1689009900,
+ 'view_count': int,
+ 'like_count': int,
+ 'repost_count': int,
+ 'comment_count': int,
+ },
+ 'expected_warnings': ['Ignoring subtitle tracks found in the HLS manifest'],
+ 'params': {'skip_download': 'm3u8'},
},
]
def _real_extract(self, url):
- video_id = self._match_id(url)
- parsed_url = compat_urlparse.urlparse(url)
- base_url = parsed_url.scheme + '://' + parsed_url.netloc
+ video_id, uploader_id = self._match_valid_url(url).group('id', 'uploader_id')
+ parsed_url = urllib.parse.urlparse(url)
+ base_url = f'{parsed_url.scheme}://{parsed_url.netloc}'
self._set_cookie(parsed_url.netloc, 'hlsPlayback', 'on')
- webpage = self._download_webpage(url, video_id)
+ full_webpage = webpage = self._download_webpage(url, video_id)
- video_url = base_url + self._html_search_regex(r'(?:<video[^>]+data-url|<source[^>]+src)="([^"]+)"', webpage, 'video url')
+ main_tweet_start = full_webpage.find('class="main-tweet"')
+ if main_tweet_start > 0:
+ webpage = full_webpage[main_tweet_start:]
+
+ video_url = '{}{}'.format(base_url, self._html_search_regex(
+ r'(?:<video[^>]+data-url|<source[^>]+src)="([^"]+)"', webpage, 'video url'))
ext = determine_ext(video_url)
if ext == 'unknown_video':
else:
formats = [{
'url': video_url,
- 'ext': ext
+ 'ext': ext,
}]
- title = (
- self._og_search_description(webpage).replace('\n', ' ')
- or self._html_search_regex(r'<div class="tweet-content[^>]+>([^<]+)</div>', webpage, 'title'))
- description = title
-
- mobj = re.match(self._VALID_URL, url)
- uploader_id = (
- mobj.group('uploader_id')
- or self._html_search_regex(r'<a class="fullname"[^>]+title="([^"]+)"', webpage, 'uploader name', fatal=False))
-
- if uploader_id:
- uploader_url = base_url + '/' + uploader_id
+ title = description = self._og_search_description(full_webpage, default=None) or self._html_search_regex(
+ r'<div class="tweet-content[^>]+>([^<]+)</div>', webpage, 'title', fatal=False)
- uploader = self._html_search_regex(r'<a class="fullname"[^>]+title="([^"]+)"', webpage, 'uploader name', fatal=False)
+ uploader_id = self._html_search_regex(
+ r'<a class="username"[^>]+title="@([^"]+)"', webpage, 'uploader id', fatal=False) or uploader_id
+ uploader = self._html_search_regex(
+ r'<a class="fullname"[^>]+title="([^"]+)"', webpage, 'uploader name', fatal=False)
if uploader:
- title = uploader + ' - ' + title
+ title = f'{uploader} - {title}'
- view_count = parse_count(self._html_search_regex(r'<span[^>]+class="icon-play[^>]*></span>\s([^<]+)</div>', webpage, 'view count', fatal=False))
- like_count = parse_count(self._html_search_regex(r'<span[^>]+class="icon-heart[^>]*></span>\s([^<]+)</div>', webpage, 'like count', fatal=False))
- repost_count = parse_count(self._html_search_regex(r'<span[^>]+class="icon-retweet[^>]*></span>\s([^<]+)</div>', webpage, 'repost count', fatal=False))
- comment_count = parse_count(self._html_search_regex(r'<span[^>]+class="icon-comment[^>]*></span>\s([^<]+)</div>', webpage, 'repost count', fatal=False))
-
- thumbnail = base_url + (self._html_search_meta('og:image', webpage, 'thumbnail url')
- or self._html_search_regex(r'<video[^>]+poster="([^"]+)"', webpage, 'thumbnail url', fatal=False))
+ counts = {
+ f'{x[0]}_count': self._html_search_regex(
+ fr'<span[^>]+class="icon-{x[1]}[^>]*></span>([^<]*)</div>',
+ webpage, f'{x[0]} count', fatal=False)
+ for x in (('view', 'play'), ('like', 'heart'), ('repost', 'retweet'), ('comment', 'comment'))
+ }
+ counts = {field: 0 if count == '' else parse_count(count) for field, count in counts.items()}
- thumbnail = remove_end(thumbnail, '%3Asmall') # if parsed with regex, it should contain this
+ thumbnail = (
+ self._html_search_meta('og:image', full_webpage, 'thumbnail url')
+ or remove_end('{}{}'.format(base_url, self._html_search_regex(
+ r'<video[^>]+poster="([^"]+)"', webpage, 'thumbnail url', fatal=False)), '%3Asmall'))
- thumbnails = []
- thumbnail_ids = ('thumb', 'small', 'large', 'medium', 'orig')
- for id in thumbnail_ids:
- thumbnails.append({
- 'id': id,
- 'url': thumbnail + '%3A' + id,
- })
+ thumbnails = [
+ {'id': id_, 'url': f'{thumbnail}%3A{id_}'}
+ for id_ in ('thumb', 'small', 'large', 'medium', 'orig')
+ ]
- date = self._html_search_regex(r'<span[^>]+class="tweet-date"[^>]*><a[^>]+title="([^"]+)"', webpage, 'upload date', fatal=False)
- upload_date = unified_strdate(date)
- timestamp = unified_timestamp(date)
+ date = self._html_search_regex(
+ r'<span[^>]+class="tweet-date"[^>]*><a[^>]+title="([^"]+)"',
+ webpage, 'upload date', default='').replace('路', '')
return {
'id': video_id,
'title': title,
'description': description,
'uploader': uploader,
- 'timestamp': timestamp,
+ 'timestamp': unified_timestamp(date),
'uploader_id': uploader_id,
- 'uploader_url': uploader_url,
- 'view_count': view_count,
- 'like_count': like_count,
- 'repost_count': repost_count,
- 'comment_count': comment_count,
+ 'uploader_url': f'{base_url}/{uploader_id}',
'formats': formats,
'thumbnails': thumbnails,
'thumbnail': thumbnail,
- 'upload_date': upload_date,
+ **counts,
}
projects / yt-dlp.git / blobdiff