[yt-dlp.git] / yt_dlp / extractor / bitchute.py

import functools
import re

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    HEADRequest,
    OnDemandPagedList,
    clean_html,
    get_element_by_class,
    get_element_by_id,
    get_elements_html_by_class,
    int_or_none,
    orderedSet,
    parse_count,
    parse_duration,
    traverse_obj,
    unified_strdate,
    urlencode_postdata,
)


class BitChuteIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?bitchute\.com/(?:video|embed|torrent/[^/]+)/(?P<id>[^/?#&]+)'
    _EMBED_REGEX = [rf'<(?:script|iframe)[^>]+\bsrc=(["\'])(?P<url>{_VALID_URL})']
    _TESTS = [{
        'url': 'https://www.bitchute.com/video/UGlrF9o9b-Q/',
        'md5': '7e427d7ed7af5a75b5855705ec750e2b',
        'info_dict': {
            'id': 'UGlrF9o9b-Q',
            'ext': 'mp4',
            'title': 'This is the first video on #BitChute !',
            'description': 'md5:a0337e7b1fe39e32336974af8173a034',
            'thumbnail': r're:^https?://.*\.jpg$',
            'uploader': 'BitChute',
            'upload_date': '20170103',
        },
    }, {
        # video not downloadable in browser, but we can recover it
        'url': 'https://www.bitchute.com/video/2s6B3nZjAk7R/',
        'md5': '05c12397d5354bf24494885b08d24ed1',
        'info_dict': {
            'id': '2s6B3nZjAk7R',
            'ext': 'mp4',
            'filesize': 71537926,
            'title': 'STYXHEXENHAMMER666 - Election Fraud, Clinton 2020, EU Armies, and Gun Control',
            'description': 'md5:228ee93bd840a24938f536aeac9cf749',
            'thumbnail': r're:^https?://.*\.jpg$',
            'uploader': 'BitChute',
            'upload_date': '20181113',
        },
        'params': {'check_formats': None},
    }, {
        # restricted video
        'url': 'https://www.bitchute.com/video/WEnQU7XGcTdl/',
        'info_dict': {
            'id': 'WEnQU7XGcTdl',
            'ext': 'mp4',
            'title': 'Impartial Truth - Ein Letzter Appell an die Vernunft',
        },
        'params': {'skip_download': True},
        'skip': 'Georestricted in DE',
    }, {
        'url': 'https://www.bitchute.com/embed/lbb5G1hjPhw/',
        'only_matching': True,
    }, {
        'url': 'https://www.bitchute.com/torrent/Zee5BE49045h/szoMrox2JEI.webtorrent',
        'only_matching': True,
    }]
    _GEO_BYPASS = False

    _HEADERS = {
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.57 Safari/537.36',
        'Referer': 'https://www.bitchute.com/',
    }

    def _check_format(self, video_url, video_id):
        urls = orderedSet(
            re.sub(r'(^https?://)(seed\d+)(?=\.bitchute\.com)', fr'\g<1>{host}', video_url)
            for host in (r'\g<2>', 'seed122', 'seed125', 'seed126', 'seed128',
                         'seed132', 'seed150', 'seed151', 'seed152', 'seed153',
                         'seed167', 'seed171', 'seed177', 'seed305', 'seed307',
                         'seedp29xb', 'zb10-7gsop1v78'))
        for url in urls:
            try:
                response = self._request_webpage(
                    HEADRequest(url), video_id=video_id, note=f'Checking {url}', headers=self._HEADERS)
            except ExtractorError as e:
                self.to_screen(f'{video_id}: URL is invalid, skipping: {e.cause}')
                continue
            return {
                'url': url,
                'filesize': int_or_none(response.headers.get('Content-Length'))
            }

    def _raise_if_restricted(self, webpage):
        page_title = clean_html(get_element_by_class('page-title', webpage)) or ''
        if re.fullmatch(r'(?:Channel|Video) Restricted', page_title):
            reason = clean_html(get_element_by_id('page-detail', webpage)) or page_title
            self.raise_geo_restricted(reason)

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(
            f'https://www.bitchute.com/video/{video_id}', video_id, headers=self._HEADERS)

        self._raise_if_restricted(webpage)
        publish_date = clean_html(get_element_by_class('video-publish-date', webpage))
        entries = self._parse_html5_media_entries(url, webpage, video_id)

        formats = []
        for format_ in traverse_obj(entries, (0, 'formats', ...)):
            if self.get_param('check_formats') is not False:
                format_.update(self._check_format(format_.pop('url'), video_id) or {})
                if 'url' not in format_:
                    continue
            formats.append(format_)

        if not formats:
            self.raise_no_formats(
                'Video is unavailable. Please make sure this video is playable in the browser '
                'before reporting this issue.', expected=True, video_id=video_id)

        return {
            'id': video_id,
            'title': self._html_extract_title(webpage) or self._og_search_title(webpage),
            'description': self._og_search_description(webpage, default=None),
            'thumbnail': self._og_search_thumbnail(webpage),
            'uploader': clean_html(get_element_by_class('owner', webpage)),
            'upload_date': unified_strdate(self._search_regex(
                r'at \d+:\d+ UTC on (.+?)\.', publish_date, 'upload date', fatal=False)),
            'formats': formats,
        }


class BitChuteChannelIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?bitchute\.com/(?P<type>channel|playlist)/(?P<id>[^/?#&]+)'
    _TESTS = [{
        'url': 'https://www.bitchute.com/channel/bitchute/',
        'info_dict': {
            'id': 'bitchute',
            'title': 'BitChute',
            'description': 'md5:5329fb3866125afa9446835594a9b138',
        },
        'playlist': [
            {
                'md5': '7e427d7ed7af5a75b5855705ec750e2b',
                'info_dict': {
                    'id': 'UGlrF9o9b-Q',
                    'ext': 'mp4',
                    'filesize': None,
                    'title': 'This is the first video on #BitChute !',
                    'description': 'md5:a0337e7b1fe39e32336974af8173a034',
                    'thumbnail': r're:^https?://.*\.jpg$',
                    'uploader': 'BitChute',
                    'upload_date': '20170103',
                    'duration': 16,
                    'view_count': int,
                },
            }
        ],
        'params': {
            'skip_download': True,
            'playlist_items': '-1',
        },
    }, {
        'url': 'https://www.bitchute.com/playlist/wV9Imujxasw9/',
        'playlist_mincount': 20,
        'info_dict': {
            'id': 'wV9Imujxasw9',
            'title': 'Bruce MacDonald and "The Light of Darkness"',
            'description': 'md5:04913227d2714af1d36d804aa2ab6b1e',
        }
    }]

    _TOKEN = 'zyG6tQcGPE5swyAEFLqKUwMuMMuF6IO2DZ6ZDQjGfsL0e4dcTLwqkTTul05Jdve7'
    PAGE_SIZE = 25
    HTML_CLASS_NAMES = {
        'channel': {
            'container': 'channel-videos-container',
            'title': 'channel-videos-title',
            'description': 'channel-videos-text',
        },
        'playlist': {
            'container': 'playlist-video',
            'title': 'title',
            'description': 'description',
        }

    }

    @staticmethod
    def _make_url(playlist_id, playlist_type):
        return f'https://www.bitchute.com/{playlist_type}/{playlist_id}/'

    def _fetch_page(self, playlist_id, playlist_type, page_num):
        playlist_url = self._make_url(playlist_id, playlist_type)
        data = self._download_json(
            f'{playlist_url}extend/', playlist_id, f'Downloading page {page_num}',
            data=urlencode_postdata({
                'csrfmiddlewaretoken': self._TOKEN,
                'name': '',
                'offset': page_num * self.PAGE_SIZE,
            }), headers={
                'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
                'Referer': playlist_url,
                'X-Requested-With': 'XMLHttpRequest',
                'Cookie': f'csrftoken={self._TOKEN}',
            })
        if not data.get('success'):
            return
        classes = self.HTML_CLASS_NAMES[playlist_type]
        for video_html in get_elements_html_by_class(classes['container'], data.get('html')):
            video_id = self._search_regex(
                r'<a\s[^>]*\bhref=["\']/video/([^"\'/]+)', video_html, 'video id', default=None)
            if not video_id:
                continue
            yield self.url_result(
                f'https://www.bitchute.com/video/{video_id}', BitChuteIE, video_id, url_transparent=True,
                title=clean_html(get_element_by_class(classes['title'], video_html)),
                description=clean_html(get_element_by_class(classes['description'], video_html)),
                duration=parse_duration(get_element_by_class('video-duration', video_html)),
                view_count=parse_count(clean_html(get_element_by_class('video-views', video_html))))

    def _real_extract(self, url):
        playlist_type, playlist_id = self._match_valid_url(url).group('type', 'id')
        webpage = self._download_webpage(self._make_url(playlist_id, playlist_type), playlist_id)

        page_func = functools.partial(self._fetch_page, playlist_id, playlist_type)
        return self.playlist_result(
            OnDemandPagedList(page_func, self.PAGE_SIZE), playlist_id,
            title=self._html_extract_title(webpage, default=None),
            description=self._html_search_meta(
                ('description', 'og:description', 'twitter:description'), webpage, default=None),
            playlist_count=int_or_none(self._html_search_regex(
                r'<span>(\d+)\s+videos?</span>', webpage, 'playlist count', default=None)))
Commit	Line	Data
c61473c1	1	import functools
b65e3b06 S	2	import re
	3
	4	from .common import InfoExtractor
6b688b89	5	from ..utils import (
37fb591c	6	ExtractorError,
f72218c1	7	HEADRequest,
c61473c1	8	OnDemandPagedList,
f72218c1	9	clean_html,
f72218c1	10	get_element_by_class,
efdc45a6	11	get_element_by_id,
c61473c1	12	get_elements_html_by_class,
f72218c1	13	int_or_none,
6b688b89	14	orderedSet,
c61473c1 M	15	parse_count,
c61473c1 M	16	parse_duration,
f72218c1	17	traverse_obj,
6ddd4bf6	18	unified_strdate,
6b688b89 S	19	urlencode_postdata,
6b688b89 S	20	)
b65e3b06 S	21
	22
	23	class BitChuteIE(InfoExtractor):
	24	_VALID_URL = r'https?://(?:www\.)?bitchute\.com/(?:video\|embed\|torrent/[^/]+)/(?P<id>[^/?#&]+)'
bfd973ec	25	_EMBED_REGEX = [rf'<(?:script\|iframe)[^>]+\bsrc=(["\'])(?P<url>{_VALID_URL})']
b65e3b06	26	_TESTS = [{
aca5774e	27	'url': 'https://www.bitchute.com/video/UGlrF9o9b-Q/',
aca5774e	28	'md5': '7e427d7ed7af5a75b5855705ec750e2b',
b65e3b06	29	'info_dict': {
f72218c1	30	'id': 'UGlrF9o9b-Q',
b65e3b06	31	'ext': 'mp4',
aca5774e	32	'title': 'This is the first video on #BitChute !',
aca5774e	33	'description': 'md5:a0337e7b1fe39e32336974af8173a034',
b65e3b06	34	'thumbnail': r're:^https?://.*\.jpg$',
aca5774e	35	'uploader': 'BitChute',
aca5774e	36	'upload_date': '20170103',
b65e3b06	37	},
f72218c1	38	}, {
	39	# video not downloadable in browser, but we can recover it
	40	'url': 'https://www.bitchute.com/video/2s6B3nZjAk7R/',
	41	'md5': '05c12397d5354bf24494885b08d24ed1',
	42	'info_dict': {
	43	'id': '2s6B3nZjAk7R',
	44	'ext': 'mp4',
	45	'filesize': 71537926,
	46	'title': 'STYXHEXENHAMMER666 - Election Fraud, Clinton 2020, EU Armies, and Gun Control',
	47	'description': 'md5:228ee93bd840a24938f536aeac9cf749',
	48	'thumbnail': r're:^https?://.*\.jpg$',
	49	'uploader': 'BitChute',
	50	'upload_date': '20181113',
	51	},
	52	'params': {'check_formats': None},
efdc45a6 M	53	}, {
	54	# restricted video
	55	'url': 'https://www.bitchute.com/video/WEnQU7XGcTdl/',
	56	'info_dict': {
	57	'id': 'WEnQU7XGcTdl',
	58	'ext': 'mp4',
	59	'title': 'Impartial Truth - Ein Letzter Appell an die Vernunft',
	60	},
	61	'params': {'skip_download': True},
	62	'skip': 'Georestricted in DE',
b65e3b06 S	63	}, {
	64	'url': 'https://www.bitchute.com/embed/lbb5G1hjPhw/',
	65	'only_matching': True,
	66	}, {
	67	'url': 'https://www.bitchute.com/torrent/Zee5BE49045h/szoMrox2JEI.webtorrent',
	68	'only_matching': True,
	69	}]
efdc45a6	70	_GEO_BYPASS = False
b65e3b06	71
f72218c1	72	_HEADERS = {
	73	'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.57 Safari/537.36',
	74	'Referer': 'https://www.bitchute.com/',
	75	}
	76
	77	def _check_format(self, video_url, video_id):
	78	urls = orderedSet(
	79	re.sub(r'(^https?://)(seed\d+)(?=\.bitchute\.com)', fr'\g<1>{host}', video_url)
0c4e0fbc N	80	for host in (r'\g<2>', 'seed122', 'seed125', 'seed126', 'seed128',
	81	'seed132', 'seed150', 'seed151', 'seed152', 'seed153',
	82	'seed167', 'seed171', 'seed177', 'seed305', 'seed307',
	83	'seedp29xb', 'zb10-7gsop1v78'))
f72218c1	84	for url in urls:
	85	try:
	86	response = self._request_webpage(
	87	HEADRequest(url), video_id=video_id, note=f'Checking {url}', headers=self._HEADERS)
	88	except ExtractorError as e:
	89	self.to_screen(f'{video_id}: URL is invalid, skipping: {e.cause}')
	90	continue
	91	return {
	92	'url': url,
	93	'filesize': int_or_none(response.headers.get('Content-Length'))
	94	}
	95
efdc45a6 M	96	def _raise_if_restricted(self, webpage):
	97	page_title = clean_html(get_element_by_class('page-title', webpage)) or ''
	98	if re.fullmatch(r'(?:Channel\|Video) Restricted', page_title):
	99	reason = clean_html(get_element_by_id('page-detail', webpage)) or page_title
	100	self.raise_geo_restricted(reason)
	101
b65e3b06 S	102	def _real_extract(self, url):
b65e3b06 S	103	video_id = self._match_id(url)
b65e3b06	104	webpage = self._download_webpage(
f72218c1	105	f'https://www.bitchute.com/video/{video_id}', video_id, headers=self._HEADERS)
b65e3b06	106
efdc45a6	107	self._raise_if_restricted(webpage)
f72218c1	108	publish_date = clean_html(get_element_by_class('video-publish-date', webpage))
f72218c1	109	entries = self._parse_html5_media_entries(url, webpage, video_id)
b65e3b06	110
f72218c1	111	formats = []
	112	for format_ in traverse_obj(entries, (0, 'formats', ...)):
	113	if self.get_param('check_formats') is not False:
	114	format_.update(self._check_format(format_.pop('url'), video_id) or {})
	115	if 'url' not in format_:
	116	continue
	117	formats.append(format_)
4c78c3d7 S	118
4c78c3d7 S	119	if not formats:
f72218c1	120	self.raise_no_formats(
	121	'Video is unavailable. Please make sure this video is playable in the browser '
	122	'before reporting this issue.', expected=True, video_id=video_id)
b65e3b06	123
b65e3b06 S	124	return {
b65e3b06 S	125	'id': video_id,
f72218c1	126	'title': self._html_extract_title(webpage) or self._og_search_title(webpage),
	127	'description': self._og_search_description(webpage, default=None),
	128	'thumbnail': self._og_search_thumbnail(webpage),
	129	'uploader': clean_html(get_element_by_class('owner', webpage)),
	130	'upload_date': unified_strdate(self._search_regex(
	131	r'at \d+:\d+ UTC on (.+?)\.', publish_date, 'upload date', fatal=False)),
b65e3b06 S	132	'formats': formats,
	133	}
	134
	135
	136	class BitChuteChannelIE(InfoExtractor):
c61473c1 M	137	_VALID_URL = r'https?://(?:www\.)?bitchute\.com/(?P<type>channel\|playlist)/(?P<id>[^/?#&]+)'
	138	_TESTS = [{
	139	'url': 'https://www.bitchute.com/channel/bitchute/',
b65e3b06	140	'info_dict': {
c61473c1 M	141	'id': 'bitchute',
	142	'title': 'BitChute',
	143	'description': 'md5:5329fb3866125afa9446835594a9b138',
b65e3b06	144	},
c61473c1 M	145	'playlist': [
	146	{
	147	'md5': '7e427d7ed7af5a75b5855705ec750e2b',
	148	'info_dict': {
	149	'id': 'UGlrF9o9b-Q',
	150	'ext': 'mp4',
	151	'filesize': None,
	152	'title': 'This is the first video on #BitChute !',
	153	'description': 'md5:a0337e7b1fe39e32336974af8173a034',
	154	'thumbnail': r're:^https?://.*\.jpg$',
	155	'uploader': 'BitChute',
	156	'upload_date': '20170103',
	157	'duration': 16,
	158	'view_count': int,
	159	},
	160	}
	161	],
	162	'params': {
	163	'skip_download': True,
	164	'playlist_items': '-1',
	165	},
	166	}, {
	167	'url': 'https://www.bitchute.com/playlist/wV9Imujxasw9/',
	168	'playlist_mincount': 20,
	169	'info_dict': {
	170	'id': 'wV9Imujxasw9',
	171	'title': 'Bruce MacDonald and "The Light of Darkness"',
	172	'description': 'md5:04913227d2714af1d36d804aa2ab6b1e',
	173	}
	174	}]
b65e3b06 S	175
b65e3b06 S	176	_TOKEN = 'zyG6tQcGPE5swyAEFLqKUwMuMMuF6IO2DZ6ZDQjGfsL0e4dcTLwqkTTul05Jdve7'
c61473c1 M	177	PAGE_SIZE = 25
	178	HTML_CLASS_NAMES = {
	179	'channel': {
	180	'container': 'channel-videos-container',
	181	'title': 'channel-videos-title',
	182	'description': 'channel-videos-text',
	183	},
	184	'playlist': {
	185	'container': 'playlist-video',
	186	'title': 'title',
	187	'description': 'description',
	188	}
	189
	190	}
b65e3b06	191
c61473c1 M	192	@staticmethod
	193	def _make_url(playlist_id, playlist_type):
	194	return f'https://www.bitchute.com/{playlist_type}/{playlist_id}/'
	195
	196	def _fetch_page(self, playlist_id, playlist_type, page_num):
	197	playlist_url = self._make_url(playlist_id, playlist_type)
	198	data = self._download_json(
	199	f'{playlist_url}extend/', playlist_id, f'Downloading page {page_num}',
	200	data=urlencode_postdata({
	201	'csrfmiddlewaretoken': self._TOKEN,
	202	'name': '',
	203	'offset': page_num * self.PAGE_SIZE,
	204	}), headers={
	205	'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
	206	'Referer': playlist_url,
	207	'X-Requested-With': 'XMLHttpRequest',
	208	'Cookie': f'csrftoken={self._TOKEN}',
	209	})
	210	if not data.get('success'):
	211	return
	212	classes = self.HTML_CLASS_NAMES[playlist_type]
	213	for video_html in get_elements_html_by_class(classes['container'], data.get('html')):
	214	video_id = self._search_regex(
	215	r'<a\s[^>]*\bhref=["\']/video/([^"\'/]+)', video_html, 'video id', default=None)
	216	if not video_id:
	217	continue
	218	yield self.url_result(
	219	f'https://www.bitchute.com/video/{video_id}', BitChuteIE, video_id, url_transparent=True,
	220	title=clean_html(get_element_by_class(classes['title'], video_html)),
	221	description=clean_html(get_element_by_class(classes['description'], video_html)),
	222	duration=parse_duration(get_element_by_class('video-duration', video_html)),
	223	view_count=parse_count(clean_html(get_element_by_class('video-views', video_html))))
b65e3b06 S	224
b65e3b06 S	225	def _real_extract(self, url):
c61473c1 M	226	playlist_type, playlist_id = self._match_valid_url(url).group('type', 'id')
	227	webpage = self._download_webpage(self._make_url(playlist_id, playlist_type), playlist_id)
	228
	229	page_func = functools.partial(self._fetch_page, playlist_id, playlist_type)
b65e3b06	230	return self.playlist_result(
c61473c1 M	231	OnDemandPagedList(page_func, self.PAGE_SIZE), playlist_id,
	232	title=self._html_extract_title(webpage, default=None),
	233	description=self._html_search_meta(
	234	('description', 'og:description', 'twitter:description'), webpage, default=None),
	235	playlist_count=int_or_none(self._html_search_regex(
	236	r'<span>(\d+)\s+videos?</span>', webpage, 'playlist count', default=None)))