[yt-dlp.git] / youtube_dl / extractor / fourtube.py

from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import (
    compat_b64decode,
    compat_str,
    compat_urllib_parse_unquote,
    compat_urlparse,
)
from ..utils import (
    int_or_none,
    parse_duration,
    parse_iso8601,
    str_to_int,
    try_get,
    unified_timestamp,
    url_or_none,
)


class FourTubeBaseIE(InfoExtractor):
    _TKN_HOST = 'tkn.kodicdn.com'

    def _extract_formats(self, url, video_id, media_id, sources):
        token_url = 'https://%s/%s/desktop/%s' % (
            self._TKN_HOST, media_id, '+'.join(sources))

        parsed_url = compat_urlparse.urlparse(url)
        tokens = self._download_json(token_url, video_id, data=b'', headers={
            'Origin': '%s://%s' % (parsed_url.scheme, parsed_url.hostname),
            'Referer': url,
        })
        formats = [{
            'url': tokens[format]['token'],
            'format_id': format + 'p',
            'resolution': format + 'p',
            'quality': int(format),
        } for format in sources]
        self._sort_formats(formats)
        return formats

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        kind, video_id, display_id = mobj.group('kind', 'id', 'display_id')

        if kind == 'm' or not display_id:
            url = self._URL_TEMPLATE % video_id

        webpage = self._download_webpage(url, video_id)

        title = self._html_search_meta('name', webpage)
        timestamp = parse_iso8601(self._html_search_meta(
            'uploadDate', webpage))
        thumbnail = self._html_search_meta('thumbnailUrl', webpage)
        uploader_id = self._html_search_regex(
            r'<a class="item-to-subscribe" href="[^"]+/(?:channel|user)s?/([^/"]+)" title="Go to [^"]+ page">',
            webpage, 'uploader id', fatal=False)
        uploader = self._html_search_regex(
            r'<a class="item-to-subscribe" href="[^"]+/(?:channel|user)s?/[^/"]+" title="Go to ([^"]+) page">',
            webpage, 'uploader', fatal=False)

        categories_html = self._search_regex(
            r'(?s)><i class="icon icon-tag"></i>\s*Categories / Tags\s*.*?<ul class="[^"]*?list[^"]*?">(.*?)</ul>',
            webpage, 'categories', fatal=False)
        categories = None
        if categories_html:
            categories = [
                c.strip() for c in re.findall(
                    r'(?s)<li><a.*?>(.*?)</a>', categories_html)]

        view_count = str_to_int(self._search_regex(
            r'<meta[^>]+itemprop="interactionCount"[^>]+content="UserPlays:([0-9,]+)">',
            webpage, 'view count', default=None))
        like_count = str_to_int(self._search_regex(
            r'<meta[^>]+itemprop="interactionCount"[^>]+content="UserLikes:([0-9,]+)">',
            webpage, 'like count', default=None))
        duration = parse_duration(self._html_search_meta('duration', webpage))

        media_id = self._search_regex(
            r'<button[^>]+data-id=(["\'])(?P<id>\d+)\1[^>]+data-quality=', webpage,
            'media id', default=None, group='id')
        sources = [
            quality
            for _, quality in re.findall(r'<button[^>]+data-quality=(["\'])(.+?)\1', webpage)]
        if not (media_id and sources):
            player_js = self._download_webpage(
                self._search_regex(
                    r'<script[^>]id=(["\'])playerembed\1[^>]+src=(["\'])(?P<url>.+?)\2',
                    webpage, 'player JS', group='url'),
                video_id, 'Downloading player JS')
            params_js = self._search_regex(
                r'\$\.ajax\(url,\ opts\);\s*\}\s*\}\)\(([0-9,\[\] ]+)\)',
                player_js, 'initialization parameters')
            params = self._parse_json('[%s]' % params_js, video_id)
            media_id = params[0]
            sources = ['%s' % p for p in params[2]]

        formats = self._extract_formats(url, video_id, media_id, sources)

        return {
            'id': video_id,
            'title': title,
            'formats': formats,
            'categories': categories,
            'thumbnail': thumbnail,
            'uploader': uploader,
            'uploader_id': uploader_id,
            'timestamp': timestamp,
            'like_count': like_count,
            'view_count': view_count,
            'duration': duration,
            'age_limit': 18,
        }


class FourTubeIE(FourTubeBaseIE):
    IE_NAME = '4tube'
    _VALID_URL = r'https?://(?:(?P<kind>www|m)\.)?4tube\.com/(?:videos|embed)/(?P<id>\d+)(?:/(?P<display_id>[^/?#&]+))?'
    _URL_TEMPLATE = 'https://www.4tube.com/videos/%s/video'
    _TESTS = [{
        'url': 'http://www.4tube.com/videos/209733/hot-babe-holly-michaels-gets-her-ass-stuffed-by-black',
        'md5': '6516c8ac63b03de06bc8eac14362db4f',
        'info_dict': {
            'id': '209733',
            'ext': 'mp4',
            'title': 'Hot Babe Holly Michaels gets her ass stuffed by black',
            'uploader': 'WCP Club',
            'uploader_id': 'wcp-club',
            'upload_date': '20131031',
            'timestamp': 1383263892,
            'duration': 583,
            'view_count': int,
            'like_count': int,
            'categories': list,
            'age_limit': 18,
        },
    }, {
        'url': 'http://www.4tube.com/embed/209733',
        'only_matching': True,
    }, {
        'url': 'http://m.4tube.com/videos/209733/hot-babe-holly-michaels-gets-her-ass-stuffed-by-black',
        'only_matching': True,
    }]


class FuxIE(FourTubeBaseIE):
    _VALID_URL = r'https?://(?:(?P<kind>www|m)\.)?fux\.com/(?:video|embed)/(?P<id>\d+)(?:/(?P<display_id>[^/?#&]+))?'
    _URL_TEMPLATE = 'https://www.fux.com/video/%s/video'
    _TESTS = [{
        'url': 'https://www.fux.com/video/195359/awesome-fucking-kitchen-ends-cum-swallow',
        'info_dict': {
            'id': '195359',
            'ext': 'mp4',
            'title': 'Awesome fucking in the kitchen ends with cum swallow',
            'uploader': 'alenci2342',
            'uploader_id': 'alenci2342',
            'upload_date': '20131230',
            'timestamp': 1388361660,
            'duration': 289,
            'view_count': int,
            'like_count': int,
            'categories': list,
            'age_limit': 18,
        },
        'params': {
            'skip_download': True,
        },
    }, {
        'url': 'https://www.fux.com/embed/195359',
        'only_matching': True,
    }, {
        'url': 'https://www.fux.com/video/195359/awesome-fucking-kitchen-ends-cum-swallow',
        'only_matching': True,
    }]


class PornTubeIE(FourTubeBaseIE):
    _VALID_URL = r'https?://(?:(?P<kind>www|m)\.)?porntube\.com/(?:videos/(?P<display_id>[^/]+)_|embed/)(?P<id>\d+)'
    _URL_TEMPLATE = 'https://www.porntube.com/videos/video_%s'
    _TKN_HOST = 'tkn.porntube.com'
    _TESTS = [{
        'url': 'https://www.porntube.com/videos/teen-couple-doing-anal_7089759',
        'info_dict': {
            'id': '7089759',
            'ext': 'mp4',
            'title': 'Teen couple doing anal',
            'uploader': 'Alexy',
            'uploader_id': '91488',
            'upload_date': '20150606',
            'timestamp': 1433595647,
            'duration': 5052,
            'view_count': int,
            'like_count': int,
            'age_limit': 18,
        },
        'params': {
            'skip_download': True,
        },
    }, {
        'url': 'https://www.porntube.com/embed/7089759',
        'only_matching': True,
    }, {
        'url': 'https://m.porntube.com/videos/teen-couple-doing-anal_7089759',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id, display_id = mobj.group('id', 'display_id')

        webpage = self._download_webpage(url, display_id)

        video = self._parse_json(
            self._search_regex(
                r'INITIALSTATE\s*=\s*(["\'])(?P<value>(?:(?!\1).)+)\1',
                webpage, 'data', group='value'), video_id,
            transform_source=lambda x: compat_urllib_parse_unquote(
                compat_b64decode(x).decode('utf-8')))['page']['video']

        title = video['title']
        media_id = video['mediaId']
        sources = [compat_str(e['height'])
                   for e in video['encodings'] if e.get('height')]
        formats = self._extract_formats(url, video_id, media_id, sources)

        thumbnail = url_or_none(video.get('masterThumb'))
        uploader = try_get(video, lambda x: x['user']['username'], compat_str)
        uploader_id = compat_str(try_get(video, lambda x: x['user']['id'], int))
        like_count = int_or_none(video.get('likes'))
        dislike_count = int_or_none(video.get('dislikes'))
        view_count = int_or_none(video.get('playsQty'))
        duration = int_or_none(video.get('durationInSeconds'))
        timestamp = unified_timestamp(video.get('publishedAt'))

        return {
            'id': video_id,
            'title': title,
            'formats': formats,
            'thumbnail': thumbnail,
            'uploader': uploader,
            'uploader_id': uploader_id,
            'timestamp': timestamp,
            'like_count': like_count,
            'dislike_count': dislike_count,
            'view_count': view_count,
            'duration': duration,
            'age_limit': 18,
        }


class PornerBrosIE(FourTubeBaseIE):
    _VALID_URL = r'https?://(?:(?P<kind>www|m)\.)?pornerbros\.com/(?:videos/(?P<display_id>[^/]+)_|embed/)(?P<id>\d+)'
    _URL_TEMPLATE = 'https://www.pornerbros.com/videos/video_%s'
    _TESTS = [{
        'url': 'https://www.pornerbros.com/videos/skinny-brunette-takes-big-cock-down-her-anal-hole_181369',
        'md5': '6516c8ac63b03de06bc8eac14362db4f',
        'info_dict': {
            'id': '181369',
            'ext': 'mp4',
            'title': 'Skinny brunette takes big cock down her anal hole',
            'uploader': 'PornerBros HD',
            'uploader_id': 'pornerbros-hd',
            'upload_date': '20130130',
            'timestamp': 1359527401,
            'duration': 1224,
            'view_count': int,
            'categories': list,
            'age_limit': 18,
        },
        'params': {
            'skip_download': True,
        },
    }, {
        'url': 'https://www.pornerbros.com/embed/181369',
        'only_matching': True,
    }, {
        'url': 'https://m.pornerbros.com/videos/skinny-brunette-takes-big-cock-down-her-anal-hole_181369',
        'only_matching': True,
    }]
Commit	Line	Data
ae6cae78 S	1	from __future__ import unicode_literals
ae6cae78 S	2
03635e2a MK	3	import re
	4
	5	from .common import InfoExtractor
15bf2ca0 S	6	from ..compat import (
	7	compat_b64decode,
	8	compat_str,
	9	compat_urllib_parse_unquote,
	10	compat_urlparse,
	11	)
1cc79574	12	from ..utils import (
15bf2ca0	13	int_or_none,
1cc79574	14	parse_duration,
9d22a7df	15	parse_iso8601,
1cc79574	16	str_to_int,
15bf2ca0 S	17	try_get,
	18	unified_timestamp,
	19	url_or_none,
ae6cae78	20	)
ae6cae78	21
03635e2a	22
b3c65153	23	class FourTubeBaseIE(InfoExtractor):
15bf2ca0 S	24	_TKN_HOST = 'tkn.kodicdn.com'
	25
	26	def _extract_formats(self, url, video_id, media_id, sources):
	27	token_url = 'https://%s/%s/desktop/%s' % (
	28	self._TKN_HOST, media_id, '+'.join(sources))
	29
	30	parsed_url = compat_urlparse.urlparse(url)
	31	tokens = self._download_json(token_url, video_id, data=b'', headers={
	32	'Origin': '%s://%s' % (parsed_url.scheme, parsed_url.hostname),
	33	'Referer': url,
	34	})
	35	formats = [{
	36	'url': tokens[format]['token'],
	37	'format_id': format + 'p',
	38	'resolution': format + 'p',
	39	'quality': int(format),
	40	} for format in sources]
	41	self._sort_formats(formats)
	42	return formats
	43
b3c65153 S	44	def _real_extract(self, url):
	45	mobj = re.match(self._VALID_URL, url)
	46	kind, video_id, display_id = mobj.group('kind', 'id', 'display_id')
03635e2a	47
b3c65153 S	48	if kind == 'm' or not display_id:
b3c65153 S	49	url = self._URL_TEMPLATE % video_id
03635e2a	50
9d22a7df	51	webpage = self._download_webpage(url, video_id)
03635e2a	52
9d22a7df PH	53	title = self._html_search_meta('name', webpage)
	54	timestamp = parse_iso8601(self._html_search_meta(
	55	'uploadDate', webpage))
	56	thumbnail = self._html_search_meta('thumbnailUrl', webpage)
	57	uploader_id = self._html_search_regex(
b3c65153	58	r'<a class="item-to-subscribe" href="[^"]+/(?:channel\|user)s?/([^/"]+)" title="Go to [^"]+ page">',
c677e49b	59	webpage, 'uploader id', fatal=False)
9d22a7df	60	uploader = self._html_search_regex(
b3c65153	61	r'<a class="item-to-subscribe" href="[^"]+/(?:channel\|user)s?/[^/"]+" title="Go to ([^"]+) page">',
c677e49b	62	webpage, 'uploader', fatal=False)
03635e2a	63
9d22a7df	64	categories_html = self._search_regex(
77afa008	65	r'(?s)><i class="icon icon-tag"></i>\sCategories / Tags\s.?<ul class="[^"]?list[^"]?">(.?)</ul>',
9d22a7df PH	66	webpage, 'categories', fatal=False)
	67	categories = None
	68	if categories_html:
	69	categories = [
	70	c.strip() for c in re.findall(
	71	r'(?s)<li><a.?>(.?)</a>', categories_html)]
ae6cae78	72
9d22a7df	73	view_count = str_to_int(self._search_regex(
77afa008	74	r'<meta[^>]+itemprop="interactionCount"[^>]+content="UserPlays:([0-9,]+)">',
b3c65153	75	webpage, 'view count', default=None))
9d22a7df	76	like_count = str_to_int(self._search_regex(
77afa008	77	r'<meta[^>]+itemprop="interactionCount"[^>]+content="UserLikes:([0-9,]+)">',
b3c65153	78	webpage, 'like count', default=None))
9d22a7df	79	duration = parse_duration(self._html_search_meta('duration', webpage))
ae6cae78	80
4356d907 S	81	media_id = self._search_regex(
	82	r'<button[^>]+data-id=(["\'])(?P<id>\d+)\1[^>]+data-quality=', webpage,
	83	'media id', default=None, group='id')
	84	sources = [
	85	quality
	86	for _, quality in re.findall(r'<button[^>]+data-quality=(["\'])(.+?)\1', webpage)]
	87	if not (media_id and sources):
	88	player_js = self._download_webpage(
	89	self._search_regex(
	90	r'<script[^>]id=(["\'])playerembed\1[^>]+src=(["\'])(?P<url>.+?)\2',
	91	webpage, 'player JS', group='url'),
	92	video_id, 'Downloading player JS')
	93	params_js = self._search_regex(
	94	r'\$\.ajax\(url,\ opts\);\s\}\s\}\)\(([0-9,\[\] ]+)\)',
	95	player_js, 'initialization parameters')
	96	params = self._parse_json('[%s]' % params_js, video_id)
	97	media_id = params[0]
	98	sources = ['%s' % p for p in params[2]]
03635e2a	99
15bf2ca0	100	formats = self._extract_formats(url, video_id, media_id, sources)
ae6cae78 S	101
ae6cae78 S	102	return {
03635e2a MK	103	'id': video_id,
	104	'title': title,
	105	'formats': formats,
9d22a7df PH	106	'categories': categories,
9d22a7df PH	107	'thumbnail': thumbnail,
ae6cae78 S	108	'uploader': uploader,
ae6cae78 S	109	'uploader_id': uploader_id,
9d22a7df PH	110	'timestamp': timestamp,
9d22a7df PH	111	'like_count': like_count,
ae6cae78 S	112	'view_count': view_count,
ae6cae78 S	113	'duration': duration,
03635e2a	114	'age_limit': 18,
5f6a1245	115	}
b3c65153 S	116
	117
	118	class FourTubeIE(FourTubeBaseIE):
	119	IE_NAME = '4tube'
	120	_VALID_URL = r'https?://(?:(?P<kind>www\|m)\.)?4tube\.com/(?:videos\|embed)/(?P<id>\d+)(?:/(?P<display_id>[^/?#&]+))?'
	121	_URL_TEMPLATE = 'https://www.4tube.com/videos/%s/video'
	122	_TESTS = [{
	123	'url': 'http://www.4tube.com/videos/209733/hot-babe-holly-michaels-gets-her-ass-stuffed-by-black',
	124	'md5': '6516c8ac63b03de06bc8eac14362db4f',
	125	'info_dict': {
	126	'id': '209733',
	127	'ext': 'mp4',
	128	'title': 'Hot Babe Holly Michaels gets her ass stuffed by black',
	129	'uploader': 'WCP Club',
	130	'uploader_id': 'wcp-club',
	131	'upload_date': '20131031',
	132	'timestamp': 1383263892,
	133	'duration': 583,
	134	'view_count': int,
	135	'like_count': int,
	136	'categories': list,
	137	'age_limit': 18,
	138	},
	139	}, {
	140	'url': 'http://www.4tube.com/embed/209733',
	141	'only_matching': True,
	142	}, {
	143	'url': 'http://m.4tube.com/videos/209733/hot-babe-holly-michaels-gets-her-ass-stuffed-by-black',
	144	'only_matching': True,
	145	}]
	146
	147
	148	class FuxIE(FourTubeBaseIE):
	149	_VALID_URL = r'https?://(?:(?P<kind>www\|m)\.)?fux\.com/(?:video\|embed)/(?P<id>\d+)(?:/(?P<display_id>[^/?#&]+))?'
	150	_URL_TEMPLATE = 'https://www.fux.com/video/%s/video'
	151	_TESTS = [{
	152	'url': 'https://www.fux.com/video/195359/awesome-fucking-kitchen-ends-cum-swallow',
	153	'info_dict': {
	154	'id': '195359',
	155	'ext': 'mp4',
	156	'title': 'Awesome fucking in the kitchen ends with cum swallow',
	157	'uploader': 'alenci2342',
	158	'uploader_id': 'alenci2342',
	159	'upload_date': '20131230',
	160	'timestamp': 1388361660,
	161	'duration': 289,
	162	'view_count': int,
	163	'like_count': int,
	164	'categories': list,
	165	'age_limit': 18,
	166	},
	167	'params': {
	168	'skip_download': True,
	169	},
	170	}, {
	171	'url': 'https://www.fux.com/embed/195359',
	172	'only_matching': True,
	173	}, {
	174	'url': 'https://www.fux.com/video/195359/awesome-fucking-kitchen-ends-cum-swallow',
	175	'only_matching': True,
	176	}]
	177
	178
	179	class PornTubeIE(FourTubeBaseIE):
180	_VALID_URL = r'https?://(?:(?P<kind>www\|m)\.)?porntube\.com/(?:videos/(?P<display_id>[^/]+)_\|embed/)(?P<id>\d+)'
181	_URL_TEMPLATE = 'https://www.porntube.com/videos/video_%s'
15bf2ca0	182	_TKN_HOST = 'tkn.porntube.com'
b3c65153 S	183	_TESTS = [{
	184	'url': 'https://www.porntube.com/videos/teen-couple-doing-anal_7089759',
	185	'info_dict': {
	186	'id': '7089759',
	187	'ext': 'mp4',
	188	'title': 'Teen couple doing anal',
	189	'uploader': 'Alexy',
15bf2ca0	190	'uploader_id': '91488',
b3c65153 S	191	'upload_date': '20150606',
	192	'timestamp': 1433595647,
	193	'duration': 5052,
	194	'view_count': int,
	195	'like_count': int,
b3c65153 S	196	'age_limit': 18,
	197	},
	198	'params': {
	199	'skip_download': True,
	200	},
	201	}, {
	202	'url': 'https://www.porntube.com/embed/7089759',
	203	'only_matching': True,
	204	}, {
	205	'url': 'https://m.porntube.com/videos/teen-couple-doing-anal_7089759',
	206	'only_matching': True,
	207	}]
	208
15bf2ca0 S	209	def _real_extract(self, url):
	210	mobj = re.match(self._VALID_URL, url)
	211	video_id, display_id = mobj.group('id', 'display_id')
	212
	213	webpage = self._download_webpage(url, display_id)
	214
	215	video = self._parse_json(
	216	self._search_regex(
	217	r'INITIALSTATE\s=\s(["\'])(?P<value>(?:(?!\1).)+)\1',
	218	webpage, 'data', group='value'), video_id,
	219	transform_source=lambda x: compat_urllib_parse_unquote(
	220	compat_b64decode(x).decode('utf-8')))['page']['video']
	221
	222	title = video['title']
	223	media_id = video['mediaId']
	224	sources = [compat_str(e['height'])
	225	for e in video['encodings'] if e.get('height')]
	226	formats = self._extract_formats(url, video_id, media_id, sources)
	227
	228	thumbnail = url_or_none(video.get('masterThumb'))
	229	uploader = try_get(video, lambda x: x['user']['username'], compat_str)
	230	uploader_id = compat_str(try_get(video, lambda x: x['user']['id'], int))
	231	like_count = int_or_none(video.get('likes'))
	232	dislike_count = int_or_none(video.get('dislikes'))
	233	view_count = int_or_none(video.get('playsQty'))
	234	duration = int_or_none(video.get('durationInSeconds'))
	235	timestamp = unified_timestamp(video.get('publishedAt'))
	236
	237	return {
	238	'id': video_id,
	239	'title': title,
	240	'formats': formats,
	241	'thumbnail': thumbnail,
	242	'uploader': uploader,
	243	'uploader_id': uploader_id,
	244	'timestamp': timestamp,
	245	'like_count': like_count,
	246	'dislike_count': dislike_count,
	247	'view_count': view_count,
	248	'duration': duration,
	249	'age_limit': 18,
	250	}
	251
b3c65153 S	252
	253	class PornerBrosIE(FourTubeBaseIE):
	254	_VALID_URL = r'https?://(?:(?P<kind>www\|m)\.)?pornerbros\.com/(?:videos/(?P<display_id>[^/]+)_\|embed/)(?P<id>\d+)'
	255	_URL_TEMPLATE = 'https://www.pornerbros.com/videos/video_%s'
	256	_TESTS = [{
	257	'url': 'https://www.pornerbros.com/videos/skinny-brunette-takes-big-cock-down-her-anal-hole_181369',
	258	'md5': '6516c8ac63b03de06bc8eac14362db4f',
	259	'info_dict': {
	260	'id': '181369',
	261	'ext': 'mp4',
	262	'title': 'Skinny brunette takes big cock down her anal hole',
	263	'uploader': 'PornerBros HD',
	264	'uploader_id': 'pornerbros-hd',
	265	'upload_date': '20130130',
	266	'timestamp': 1359527401,
	267	'duration': 1224,
	268	'view_count': int,
	269	'categories': list,
	270	'age_limit': 18,
	271	},
	272	'params': {
	273	'skip_download': True,
	274	},
	275	}, {
	276	'url': 'https://www.pornerbros.com/embed/181369',
	277	'only_matching': True,
	278	}, {
	279	'url': 'https://m.pornerbros.com/videos/skinny-brunette-takes-big-cock-down-her-anal-hole_181369',
	280	'only_matching': True,
	281	}]