[yt-dlp.git] / yt_dlp / extractor / curiositystream.py

import re
import urllib.parse

from .common import InfoExtractor
from ..compat import compat_str
from ..utils import ExtractorError, int_or_none, urlencode_postdata


class CuriosityStreamBaseIE(InfoExtractor):
    _NETRC_MACHINE = 'curiositystream'
    _auth_token = None

    def _handle_errors(self, result):
        error = result.get('error', {}).get('message')
        if error:
            if isinstance(error, dict):
                error = ', '.join(error.values())
            raise ExtractorError(
                '%s said: %s' % (self.IE_NAME, error), expected=True)

    def _call_api(self, path, video_id, query=None):
        headers = {}
        if not self._auth_token:
            auth_cookie = self._get_cookies('https://curiositystream.com').get('auth_token')
            if auth_cookie:
                self.write_debug('Obtained auth_token cookie')
                self._auth_token = urllib.parse.unquote(auth_cookie.value)
        if self._auth_token:
            headers['X-Auth-Token'] = self._auth_token
        result = self._download_json(
            self._API_BASE_URL + path, video_id, headers=headers, query=query)
        self._handle_errors(result)
        return result['data']

    def _perform_login(self, username, password):
        result = self._download_json(
            'https://api.curiositystream.com/v1/login', None,
            note='Logging in', data=urlencode_postdata({
                'email': username,
                'password': password,
            }))
        self._handle_errors(result)
        CuriosityStreamBaseIE._auth_token = result['message']['auth_token']


class CuriosityStreamIE(CuriosityStreamBaseIE):
    IE_NAME = 'curiositystream'
    _VALID_URL = r'https?://(?:app\.)?curiositystream\.com/video/(?P<id>\d+)'
    _TESTS = [{
        'url': 'http://app.curiositystream.com/video/2',
        'info_dict': {
            'id': '2',
            'ext': 'mp4',
            'title': 'How Did You Develop The Internet?',
            'description': 'Vint Cerf, Google\'s Chief Internet Evangelist, describes how he and Bob Kahn created the internet.',
            'channel': 'Curiosity Stream',
            'categories': ['Technology', 'Interview'],
            'average_rating': float,
            'series_id': '2',
            'thumbnail': r're:https://img.curiositystream.com/.+\.jpg',
            'tags': [],
            'duration': 158
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }]

    _API_BASE_URL = 'https://api.curiositystream.com/v1/media/'

    def _real_extract(self, url):
        video_id = self._match_id(url)

        formats = []
        for encoding_format in ('m3u8', 'mpd'):
            media = self._call_api(video_id, video_id, query={
                'encodingsNew': 'true',
                'encodingsFormat': encoding_format,
            })
            for encoding in media.get('encodings', []):
                playlist_url = encoding.get('master_playlist_url')
                if encoding_format == 'm3u8':
                    # use `m3u8` entry_protocol until EXT-X-MAP is properly supported by `m3u8_native` entry_protocol
                    formats.extend(self._extract_m3u8_formats(
                        playlist_url, video_id, 'mp4',
                        m3u8_id='hls', fatal=False))
                elif encoding_format == 'mpd':
                    formats.extend(self._extract_mpd_formats(
                        playlist_url, video_id, mpd_id='dash', fatal=False))
                encoding_url = encoding.get('url')
                file_url = encoding.get('file_url')
                if not encoding_url and not file_url:
                    continue
                f = {
                    'width': int_or_none(encoding.get('width')),
                    'height': int_or_none(encoding.get('height')),
                    'vbr': int_or_none(encoding.get('video_bitrate')),
                    'abr': int_or_none(encoding.get('audio_bitrate')),
                    'filesize': int_or_none(encoding.get('size_in_bytes')),
                    'vcodec': encoding.get('video_codec'),
                    'acodec': encoding.get('audio_codec'),
                    'container': encoding.get('container_type'),
                }
                for f_url in (encoding_url, file_url):
                    if not f_url:
                        continue
                    fmt = f.copy()
                    rtmp = re.search(r'^(?P<url>rtmpe?://(?P<host>[^/]+)/(?P<app>.+))/(?P<playpath>mp[34]:.+)$', f_url)
                    if rtmp:
                        fmt.update({
                            'url': rtmp.group('url'),
                            'play_path': rtmp.group('playpath'),
                            'app': rtmp.group('app'),
                            'ext': 'flv',
                            'format_id': 'rtmp',
                        })
                    else:
                        fmt.update({
                            'url': f_url,
                            'format_id': 'http',
                        })
                    formats.append(fmt)

        title = media['title']

        subtitles = {}
        for closed_caption in media.get('closed_captions', []):
            sub_url = closed_caption.get('file')
            if not sub_url:
                continue
            lang = closed_caption.get('code') or closed_caption.get('language') or 'en'
            subtitles.setdefault(lang, []).append({
                'url': sub_url,
            })

        return {
            'id': video_id,
            'formats': formats,
            'title': title,
            'description': media.get('description'),
            'thumbnail': media.get('image_large') or media.get('image_medium') or media.get('image_small'),
            'duration': int_or_none(media.get('duration')),
            'tags': media.get('tags'),
            'subtitles': subtitles,
            'channel': media.get('producer'),
            'categories': [media.get('primary_category'), media.get('type')],
            'average_rating': media.get('rating_percentage'),
            'series_id': str(media.get('collection_id') or '') or None,
        }


class CuriosityStreamCollectionBaseIE(CuriosityStreamBaseIE):

    def _real_extract(self, url):
        collection_id = self._match_id(url)
        collection = self._call_api(collection_id, collection_id)
        entries = []
        for media in collection.get('media', []):
            media_id = compat_str(media.get('id'))
            media_type, ie = ('series', CuriosityStreamSeriesIE) if media.get('is_collection') else ('video', CuriosityStreamIE)
            entries.append(self.url_result(
                'https://curiositystream.com/%s/%s' % (media_type, media_id),
                ie=ie.ie_key(), video_id=media_id))
        return self.playlist_result(
            entries, collection_id,
            collection.get('title'), collection.get('description'))


class CuriosityStreamCollectionsIE(CuriosityStreamCollectionBaseIE):
    IE_NAME = 'curiositystream:collections'
    _VALID_URL = r'https?://(?:app\.)?curiositystream\.com/collections/(?P<id>\d+)'
    _API_BASE_URL = 'https://api.curiositystream.com/v2/collections/'
    _TESTS = [{
        'url': 'https://curiositystream.com/collections/86',
        'info_dict': {
            'id': '86',
            'title': 'Staff Picks',
            'description': 'Wondering where to start? Here are a few of our favorite series and films... from our couch to yours.',
        },
        'playlist_mincount': 7,
    }, {
        'url': 'https://curiositystream.com/collections/36',
        'only_matching': True,
    }]


class CuriosityStreamSeriesIE(CuriosityStreamCollectionBaseIE):
    IE_NAME = 'curiositystream:series'
    _VALID_URL = r'https?://(?:app\.)?curiositystream\.com/(?:series|collection)/(?P<id>\d+)'
    _API_BASE_URL = 'https://api.curiositystream.com/v2/series/'
    _TESTS = [{
        'url': 'https://curiositystream.com/series/2',
        'info_dict': {
            'id': '2',
            'title': 'Curious Minds: The Internet',
            'description': 'How is the internet shaping our lives in the 21st Century?',
        },
        'playlist_mincount': 16,
    }, {
        'url': 'https://curiositystream.com/collection/2',
        'only_matching': True,
    }]
Commit	Line	Data
a206ef62	1	import re
6b71d186	2	import urllib.parse
a206ef62	3
f096ec26	4	from .common import InfoExtractor
14f25df2	5	from ..compat import compat_str
14f25df2	6	from ..utils import ExtractorError, int_or_none, urlencode_postdata
f096ec26 RA	7
	8
	9	class CuriosityStreamBaseIE(InfoExtractor):
	10	_NETRC_MACHINE = 'curiositystream'
	11	_auth_token = None
f096ec26 RA	12
	13	def _handle_errors(self, result):
	14	error = result.get('error', {}).get('message')
	15	if error:
	16	if isinstance(error, dict):
	17	error = ', '.join(error.values())
	18	raise ExtractorError(
	19	'%s said: %s' % (self.IE_NAME, error), expected=True)
	20
f7ad7160	21	def _call_api(self, path, video_id, query=None):
f096ec26	22	headers = {}
d2ff2c91	23	if not self._auth_token:
	24	auth_cookie = self._get_cookies('https://curiositystream.com').get('auth_token')
	25	if auth_cookie:
	26	self.write_debug('Obtained auth_token cookie')
6b71d186	27	self._auth_token = urllib.parse.unquote(auth_cookie.value)
f096ec26 RA	28	if self._auth_token:
	29	headers['X-Auth-Token'] = self._auth_token
	30	result = self._download_json(
f7ad7160	31	self._API_BASE_URL + path, video_id, headers=headers, query=query)
f096ec26 RA	32	self._handle_errors(result)
	33	return result['data']
	34
52efa4b3	35	def _perform_login(self, username, password):
b207d5eb	36	result = self._download_json(
d0e6121a	37	'https://api.curiositystream.com/v1/login', None,
d0e6121a	38	note='Logging in', data=urlencode_postdata({
52efa4b3	39	'email': username,
b207d5eb RA	40	'password': password,
	41	}))
	42	self._handle_errors(result)
92775d8a	43	CuriosityStreamBaseIE._auth_token = result['message']['auth_token']
f096ec26	44
3b983ee4 RA	45
	46	class CuriosityStreamIE(CuriosityStreamBaseIE):
	47	IE_NAME = 'curiositystream'
	48	_VALID_URL = r'https?://(?:app\.)?curiositystream\.com/video/(?P<id>\d+)'
9ac24e23	49	_TESTS = [{
14f25df2	50	'url': 'http://app.curiositystream.com/video/2',
3b983ee4 RA	51	'info_dict': {
	52	'id': '2',
	53	'ext': 'mp4',
	54	'title': 'How Did You Develop The Internet?',
	55	'description': 'Vint Cerf, Google\'s Chief Internet Evangelist, describes how he and Bob Kahn created the internet.',
9ac24e23	56	'channel': 'Curiosity Stream',
9ac24e23	57	'categories': ['Technology', 'Interview'],
6b71d186	58	'average_rating': float,
9ac24e23	59	'series_id': '2',
6b71d186	60	'thumbnail': r're:https://img.curiositystream.com/.+\.jpg',
	61	'tags': [],
	62	'duration': 158
f7ad7160	63	},
f7ad7160	64	'params': {
f7ad7160	65	# m3u8 download
	66	'skip_download': True,
	67	},
9ac24e23	68	}]
3b983ee4	69
d0e6121a	70	_API_BASE_URL = 'https://api.curiositystream.com/v1/media/'
d0e6121a	71
3b983ee4 RA	72	def _real_extract(self, url):
3b983ee4 RA	73	video_id = self._match_id(url)
f096ec26	74
a206ef62	75	formats = []
f7ad7160	76	for encoding_format in ('m3u8', 'mpd'):
d0e6121a	77	media = self._call_api(video_id, video_id, query={
f7ad7160	78	'encodingsNew': 'true',
	79	'encodingsFormat': encoding_format,
	80	})
	81	for encoding in media.get('encodings', []):
	82	playlist_url = encoding.get('master_playlist_url')
	83	if encoding_format == 'm3u8':
	84	# use `m3u8` entry_protocol until EXT-X-MAP is properly supported by `m3u8_native` entry_protocol
	85	formats.extend(self._extract_m3u8_formats(
	86	playlist_url, video_id, 'mp4',
	87	m3u8_id='hls', fatal=False))
	88	elif encoding_format == 'mpd':
	89	formats.extend(self._extract_mpd_formats(
	90	playlist_url, video_id, mpd_id='dash', fatal=False))
	91	encoding_url = encoding.get('url')
	92	file_url = encoding.get('file_url')
	93	if not encoding_url and not file_url:
a206ef62	94	continue
f7ad7160	95	f = {
	96	'width': int_or_none(encoding.get('width')),
	97	'height': int_or_none(encoding.get('height')),
	98	'vbr': int_or_none(encoding.get('video_bitrate')),
	99	'abr': int_or_none(encoding.get('audio_bitrate')),
	100	'filesize': int_or_none(encoding.get('size_in_bytes')),
	101	'vcodec': encoding.get('video_codec'),
	102	'acodec': encoding.get('audio_codec'),
	103	'container': encoding.get('container_type'),
	104	}
	105	for f_url in (encoding_url, file_url):
	106	if not f_url:
	107	continue
	108	fmt = f.copy()
	109	rtmp = re.search(r'^(?P<url>rtmpe?://(?P<host>[^/]+)/(?P<app>.+))/(?P<playpath>mp[34]:.+)$', f_url)
	110	if rtmp:
	111	fmt.update({
	112	'url': rtmp.group('url'),
	113	'play_path': rtmp.group('playpath'),
	114	'app': rtmp.group('app'),
	115	'ext': 'flv',
	116	'format_id': 'rtmp',
	117	})
	118	else:
	119	fmt.update({
	120	'url': f_url,
	121	'format_id': 'http',
	122	})
	123	formats.append(fmt)
a206ef62	124
f7ad7160	125	title = media['title']
f7ad7160	126
f096ec26 RA	127	subtitles = {}
	128	for closed_caption in media.get('closed_captions', []):
	129	sub_url = closed_caption.get('file')
	130	if not sub_url:
	131	continue
	132	lang = closed_caption.get('code') or closed_caption.get('language') or 'en'
	133	subtitles.setdefault(lang, []).append({
	134	'url': sub_url,
	135	})
	136
	137	return {
f096ec26	138	'id': video_id,
a206ef62	139	'formats': formats,
f096ec26 RA	140	'title': title,
	141	'description': media.get('description'),
	142	'thumbnail': media.get('image_large') or media.get('image_medium') or media.get('image_small'),
	143	'duration': int_or_none(media.get('duration')),
	144	'tags': media.get('tags'),
	145	'subtitles': subtitles,
9ac24e23	146	'channel': media.get('producer'),
	147	'categories': [media.get('primary_category'), media.get('type')],
	148	'average_rating': media.get('rating_percentage'),
	149	'series_id': str(media.get('collection_id') or '') or None,
f096ec26 RA	150	}
	151
	152
92775d8a	153	class CuriosityStreamCollectionBaseIE(CuriosityStreamBaseIE):
	154
	155	def _real_extract(self, url):
	156	collection_id = self._match_id(url)
	157	collection = self._call_api(collection_id, collection_id)
	158	entries = []
	159	for media in collection.get('media', []):
	160	media_id = compat_str(media.get('id'))
	161	media_type, ie = ('series', CuriosityStreamSeriesIE) if media.get('is_collection') else ('video', CuriosityStreamIE)
	162	entries.append(self.url_result(
	163	'https://curiositystream.com/%s/%s' % (media_type, media_id),
	164	ie=ie.ie_key(), video_id=media_id))
	165	return self.playlist_result(
	166	entries, collection_id,
	167	collection.get('title'), collection.get('description'))
	168
	169
	170	class CuriosityStreamCollectionsIE(CuriosityStreamCollectionBaseIE):
	171	IE_NAME = 'curiositystream:collections'
	172	_VALID_URL = r'https?://(?:app\.)?curiositystream\.com/collections/(?P<id>\d+)'
ef39f860	173	_API_BASE_URL = 'https://api.curiositystream.com/v2/collections/'
3b983ee4	174	_TESTS = [{
ef39f860	175	'url': 'https://curiositystream.com/collections/86',
f096ec26	176	'info_dict': {
ef39f860	177	'id': '86',
	178	'title': 'Staff Picks',
	179	'description': 'Wondering where to start? Here are a few of our favorite series and films... from our couch to yours.',
f096ec26	180	},
ef39f860	181	'playlist_mincount': 7,
ed807c18	182	}, {
92775d8a	183	'url': 'https://curiositystream.com/collections/36',
	184	'only_matching': True,
	185	}]
	186
	187
	188	class CuriosityStreamSeriesIE(CuriosityStreamCollectionBaseIE):
	189	IE_NAME = 'curiositystream:series'
	190	_VALID_URL = r'https?://(?:app\.)?curiositystream\.com/(?:series\|collection)/(?P<id>\d+)'
	191	_API_BASE_URL = 'https://api.curiositystream.com/v2/series/'
	192	_TESTS = [{
	193	'url': 'https://curiositystream.com/series/2',
ed807c18	194	'info_dict': {
	195	'id': '2',
	196	'title': 'Curious Minds: The Internet',
	197	'description': 'How is the internet shaping our lives in the 21st Century?',
	198	},
	199	'playlist_mincount': 16,
	200	}, {
92775d8a	201	'url': 'https://curiositystream.com/collection/2',
ed807c18	202	'only_matching': True,
3b983ee4	203	}]