[yt-dlp.git] / youtube_dl / extractor / twitch.py

# coding: utf-8
from __future__ import unicode_literals

import itertools
import re
import random

from .common import InfoExtractor
from ..compat import (
    compat_HTTPError,
    compat_parse_qs,
    compat_str,
    compat_urllib_parse_urlencode,
    compat_urllib_parse_urlparse,
    compat_urlparse,
)
from ..utils import (
    clean_html,
    ExtractorError,
    int_or_none,
    js_to_json,
    orderedSet,
    parse_duration,
    parse_iso8601,
    update_url_query,
    urlencode_postdata,
)


class TwitchBaseIE(InfoExtractor):
    _VALID_URL_BASE = r'https?://(?:www\.)?twitch\.tv'

    _API_BASE = 'https://api.twitch.tv'
    _USHER_BASE = 'https://usher.ttvnw.net'
    _LOGIN_URL = 'http://www.twitch.tv/login'
    _CLIENT_ID = 'jzkbprff40iqj646a697cyrvl0zt2m6'
    _NETRC_MACHINE = 'twitch'

    def _handle_error(self, response):
        if not isinstance(response, dict):
            return
        error = response.get('error')
        if error:
            raise ExtractorError(
                '%s returned error: %s - %s' % (self.IE_NAME, error, response.get('message')),
                expected=True)

    def _call_api(self, path, item_id, note):
        response = self._download_json(
            '%s/%s' % (self._API_BASE, path), item_id, note,
            headers={'Client-ID': self._CLIENT_ID})
        self._handle_error(response)
        return response

    def _real_initialize(self):
        self._login()

    def _login(self):
        (username, password) = self._get_login_info()
        if username is None:
            return

        def fail(message):
            raise ExtractorError(
                'Unable to login. Twitch said: %s' % message, expected=True)

        login_page, handle = self._download_webpage_handle(
            self._LOGIN_URL, None, 'Downloading login page')

        # Some TOR nodes and public proxies are blocked completely
        if 'blacklist_message' in login_page:
            fail(clean_html(login_page))

        login_form = self._hidden_inputs(login_page)

        login_form.update({
            'username': username,
            'password': password,
        })

        redirect_url = handle.geturl()

        post_url = self._search_regex(
            r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page,
            'post url', default=redirect_url, group='url')

        if not post_url.startswith('http'):
            post_url = compat_urlparse.urljoin(redirect_url, post_url)

        headers = {'Referer': redirect_url}

        try:
            response = self._download_json(
                post_url, None, 'Logging in as %s' % username,
                data=urlencode_postdata(login_form),
                headers=headers)
        except ExtractorError as e:
            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 400:
                response = self._parse_json(
                    e.cause.read().decode('utf-8'), None)
                fail(response['message'])
            raise

        if response.get('redirect'):
            self._download_webpage(
                response['redirect'], None, 'Downloading login redirect page',
                headers=headers)

    def _prefer_source(self, formats):
        try:
            source = next(f for f in formats if f['format_id'] == 'Source')
            source['preference'] = 10
        except StopIteration:
            pass  # No Source stream present
        self._sort_formats(formats)


class TwitchItemBaseIE(TwitchBaseIE):
    def _download_info(self, item, item_id):
        return self._extract_info(self._call_api(
            'kraken/videos/%s%s' % (item, item_id), item_id,
            'Downloading %s info JSON' % self._ITEM_TYPE))

    def _extract_media(self, item_id):
        info = self._download_info(self._ITEM_SHORTCUT, item_id)
        response = self._call_api(
            'api/videos/%s%s' % (self._ITEM_SHORTCUT, item_id), item_id,
            'Downloading %s playlist JSON' % self._ITEM_TYPE)
        entries = []
        chunks = response['chunks']
        qualities = list(chunks.keys())
        for num, fragment in enumerate(zip(*chunks.values()), start=1):
            formats = []
            for fmt_num, fragment_fmt in enumerate(fragment):
                format_id = qualities[fmt_num]
                fmt = {
                    'url': fragment_fmt['url'],
                    'format_id': format_id,
                    'quality': 1 if format_id == 'live' else 0,
                }
                m = re.search(r'^(?P<height>\d+)[Pp]', format_id)
                if m:
                    fmt['height'] = int(m.group('height'))
                formats.append(fmt)
            self._sort_formats(formats)
            entry = dict(info)
            entry['id'] = '%s_%d' % (entry['id'], num)
            entry['title'] = '%s part %d' % (entry['title'], num)
            entry['formats'] = formats
            entries.append(entry)
        return self.playlist_result(entries, info['id'], info['title'])

    def _extract_info(self, info):
        return {
            'id': info['_id'],
            'title': info.get('title') or 'Untitled Broadcast',
            'description': info.get('description'),
            'duration': int_or_none(info.get('length')),
            'thumbnail': info.get('preview'),
            'uploader': info.get('channel', {}).get('display_name'),
            'uploader_id': info.get('channel', {}).get('name'),
            'timestamp': parse_iso8601(info.get('recorded_at')),
            'view_count': int_or_none(info.get('views')),
        }

    def _real_extract(self, url):
        return self._extract_media(self._match_id(url))


class TwitchVideoIE(TwitchItemBaseIE):
    IE_NAME = 'twitch:video'
    _VALID_URL = r'%s/[^/]+/b/(?P<id>\d+)' % TwitchBaseIE._VALID_URL_BASE
    _ITEM_TYPE = 'video'
    _ITEM_SHORTCUT = 'a'

    _TEST = {
        'url': 'http://www.twitch.tv/riotgames/b/577357806',
        'info_dict': {
            'id': 'a577357806',
            'title': 'Worlds Semifinals - Star Horn Royal Club vs. OMG',
        },
        'playlist_mincount': 12,
        'skip': 'HTTP Error 404: Not Found',
    }


class TwitchChapterIE(TwitchItemBaseIE):
    IE_NAME = 'twitch:chapter'
    _VALID_URL = r'%s/[^/]+/c/(?P<id>\d+)' % TwitchBaseIE._VALID_URL_BASE
    _ITEM_TYPE = 'chapter'
    _ITEM_SHORTCUT = 'c'

    _TESTS = [{
        'url': 'http://www.twitch.tv/acracingleague/c/5285812',
        'info_dict': {
            'id': 'c5285812',
            'title': 'ACRL Off Season - Sports Cars @ Nordschleife',
        },
        'playlist_mincount': 3,
        'skip': 'HTTP Error 404: Not Found',
    }, {
        'url': 'http://www.twitch.tv/tsm_theoddone/c/2349361',
        'only_matching': True,
    }]


class TwitchVodIE(TwitchItemBaseIE):
    IE_NAME = 'twitch:vod'
    _VALID_URL = r'''(?x)
                    https?://
                        (?:
                            (?:www\.)?twitch\.tv/(?:[^/]+/v|videos)/|
                            player\.twitch\.tv/\?.*?\bvideo=v
                        )
                        (?P<id>\d+)
                    '''
    _ITEM_TYPE = 'vod'
    _ITEM_SHORTCUT = 'v'

    _TESTS = [{
        'url': 'http://www.twitch.tv/riotgames/v/6528877?t=5m10s',
        'info_dict': {
            'id': 'v6528877',
            'ext': 'mp4',
            'title': 'LCK Summer Split - Week 6 Day 1',
            'thumbnail': r're:^https?://.*\.jpg$',
            'duration': 17208,
            'timestamp': 1435131709,
            'upload_date': '20150624',
            'uploader': 'Riot Games',
            'uploader_id': 'riotgames',
            'view_count': int,
            'start_time': 310,
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }, {
        # Untitled broadcast (title is None)
        'url': 'http://www.twitch.tv/belkao_o/v/11230755',
        'info_dict': {
            'id': 'v11230755',
            'ext': 'mp4',
            'title': 'Untitled Broadcast',
            'thumbnail': r're:^https?://.*\.jpg$',
            'duration': 1638,
            'timestamp': 1439746708,
            'upload_date': '20150816',
            'uploader': 'BelkAO_o',
            'uploader_id': 'belkao_o',
            'view_count': int,
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
        'skip': 'HTTP Error 404: Not Found',
    }, {
        'url': 'http://player.twitch.tv/?t=5m10s&video=v6528877',
        'only_matching': True,
    }, {
        'url': 'https://www.twitch.tv/videos/6528877',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        item_id = self._match_id(url)

        info = self._download_info(self._ITEM_SHORTCUT, item_id)
        access_token = self._call_api(
            'api/vods/%s/access_token' % item_id, item_id,
            'Downloading %s access token' % self._ITEM_TYPE)

        formats = self._extract_m3u8_formats(
            '%s/vod/%s?%s' % (
                self._USHER_BASE, item_id,
                compat_urllib_parse_urlencode({
                    'allow_source': 'true',
                    'allow_audio_only': 'true',
                    'allow_spectre': 'true',
                    'player': 'twitchweb',
                    'nauth': access_token['token'],
                    'nauthsig': access_token['sig'],
                })),
            item_id, 'mp4', entry_protocol='m3u8_native')

        self._prefer_source(formats)
        info['formats'] = formats

        parsed_url = compat_urllib_parse_urlparse(url)
        query = compat_parse_qs(parsed_url.query)
        if 't' in query:
            info['start_time'] = parse_duration(query['t'][0])

        if info.get('timestamp') is not None:
            info['subtitles'] = {
                'rechat': [{
                    'url': update_url_query(
                        'https://rechat.twitch.tv/rechat-messages', {
                            'video_id': 'v%s' % item_id,
                            'start': info['timestamp'],
                        }),
                    'ext': 'json',
                }],
            }

        return info


class TwitchPlaylistBaseIE(TwitchBaseIE):
    _PLAYLIST_PATH = 'kraken/channels/%s/videos/?offset=%d&limit=%d'
    _PAGE_LIMIT = 100

    def _extract_playlist(self, channel_id):
        info = self._call_api(
            'kraken/channels/%s' % channel_id,
            channel_id, 'Downloading channel info JSON')
        channel_name = info.get('display_name') or info.get('name')
        entries = []
        offset = 0
        limit = self._PAGE_LIMIT
        broken_paging_detected = False
        counter_override = None
        for counter in itertools.count(1):
            response = self._call_api(
                self._PLAYLIST_PATH % (channel_id, offset, limit),
                channel_id,
                'Downloading %s JSON page %s'
                % (self._PLAYLIST_TYPE, counter_override or counter))
            page_entries = self._extract_playlist_page(response)
            if not page_entries:
                break
            total = int_or_none(response.get('_total'))
            # Since the beginning of March 2016 twitch's paging mechanism
            # is completely broken on the twitch side. It simply ignores
            # a limit and returns the whole offset number of videos.
            # Working around by just requesting all videos at once.
            # Upd: pagination bug was fixed by twitch on 15.03.2016.
            if not broken_paging_detected and total and len(page_entries) > limit:
                self.report_warning(
                    'Twitch pagination is broken on twitch side, requesting all videos at once',
                    channel_id)
                broken_paging_detected = True
                offset = total
                counter_override = '(all at once)'
                continue
            entries.extend(page_entries)
            if broken_paging_detected or total and len(page_entries) >= total:
                break
            offset += limit
        return self.playlist_result(
            [self.url_result(entry) for entry in orderedSet(entries)],
            channel_id, channel_name)

    def _extract_playlist_page(self, response):
        videos = response.get('videos')
        return [video['url'] for video in videos] if videos else []

    def _real_extract(self, url):
        return self._extract_playlist(self._match_id(url))


class TwitchProfileIE(TwitchPlaylistBaseIE):
    IE_NAME = 'twitch:profile'
    _VALID_URL = r'%s/(?P<id>[^/]+)/profile/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
    _PLAYLIST_TYPE = 'profile'

    _TEST = {
        'url': 'http://www.twitch.tv/vanillatv/profile',
        'info_dict': {
            'id': 'vanillatv',
            'title': 'VanillaTV',
        },
        'playlist_mincount': 412,
    }


class TwitchVideosBaseIE(TwitchPlaylistBaseIE):
    _VALID_URL_VIDEOS_BASE = r'%s/(?P<id>[^/]+)/videos' % TwitchBaseIE._VALID_URL_BASE
    _PLAYLIST_PATH = TwitchPlaylistBaseIE._PLAYLIST_PATH + '&broadcast_type='


class TwitchAllVideosIE(TwitchVideosBaseIE):
    IE_NAME = 'twitch:videos:all'
    _VALID_URL = r'%s/all' % TwitchVideosBaseIE._VALID_URL_VIDEOS_BASE
    _PLAYLIST_PATH = TwitchVideosBaseIE._PLAYLIST_PATH + 'archive,upload,highlight'
    _PLAYLIST_TYPE = 'all videos'

    _TEST = {
        'url': 'https://www.twitch.tv/spamfish/videos/all',
        'info_dict': {
            'id': 'spamfish',
            'title': 'Spamfish',
        },
        'playlist_mincount': 869,
    }


class TwitchUploadsIE(TwitchVideosBaseIE):
    IE_NAME = 'twitch:videos:uploads'
    _VALID_URL = r'%s/uploads' % TwitchVideosBaseIE._VALID_URL_VIDEOS_BASE
    _PLAYLIST_PATH = TwitchVideosBaseIE._PLAYLIST_PATH + 'upload'
    _PLAYLIST_TYPE = 'uploads'

    _TEST = {
        'url': 'https://www.twitch.tv/spamfish/videos/uploads',
        'info_dict': {
            'id': 'spamfish',
            'title': 'Spamfish',
        },
        'playlist_mincount': 0,
    }


class TwitchPastBroadcastsIE(TwitchVideosBaseIE):
    IE_NAME = 'twitch:videos:past-broadcasts'
    _VALID_URL = r'%s/past-broadcasts' % TwitchVideosBaseIE._VALID_URL_VIDEOS_BASE
    _PLAYLIST_PATH = TwitchVideosBaseIE._PLAYLIST_PATH + 'archive'
    _PLAYLIST_TYPE = 'past broadcasts'

    _TEST = {
        'url': 'https://www.twitch.tv/spamfish/videos/past-broadcasts',
        'info_dict': {
            'id': 'spamfish',
            'title': 'Spamfish',
        },
        'playlist_mincount': 0,
    }


class TwitchHighlightsIE(TwitchVideosBaseIE):
    IE_NAME = 'twitch:videos:highlights'
    _VALID_URL = r'%s/highlights' % TwitchVideosBaseIE._VALID_URL_VIDEOS_BASE
    _PLAYLIST_PATH = TwitchVideosBaseIE._PLAYLIST_PATH + 'highlight'
    _PLAYLIST_TYPE = 'highlights'

    _TEST = {
        'url': 'https://www.twitch.tv/spamfish/videos/highlights',
        'info_dict': {
            'id': 'spamfish',
            'title': 'Spamfish',
        },
        'playlist_mincount': 805,
    }


class TwitchStreamIE(TwitchBaseIE):
    IE_NAME = 'twitch:stream'
    _VALID_URL = r'''(?x)
                    https?://
                        (?:
                            (?:www\.)?twitch\.tv/|
                            player\.twitch\.tv/\?.*?\bchannel=
                        )
                        (?P<id>[^/#?]+)
                    '''

    _TESTS = [{
        'url': 'http://www.twitch.tv/shroomztv',
        'info_dict': {
            'id': '12772022048',
            'display_id': 'shroomztv',
            'ext': 'mp4',
            'title': 're:^ShroomzTV [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
            'description': 'H1Z1 - lonewolfing with ShroomzTV | A3 Battle Royale later - @ShroomzTV',
            'is_live': True,
            'timestamp': 1421928037,
            'upload_date': '20150122',
            'uploader': 'ShroomzTV',
            'uploader_id': 'shroomztv',
            'view_count': int,
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }, {
        'url': 'http://www.twitch.tv/miracle_doto#profile-0',
        'only_matching': True,
    }, {
        'url': 'https://player.twitch.tv/?channel=lotsofs',
        'only_matching': True,
    }]

    @classmethod
    def suitable(cls, url):
        return (False
                if any(ie.suitable(url) for ie in (
                    TwitchVideoIE,
                    TwitchChapterIE,
                    TwitchVodIE,
                    TwitchProfileIE,
                    TwitchAllVideosIE,
                    TwitchUploadsIE,
                    TwitchPastBroadcastsIE,
                    TwitchHighlightsIE))
                else super(TwitchStreamIE, cls).suitable(url))

    def _real_extract(self, url):
        channel_id = self._match_id(url)

        stream = self._call_api(
            'kraken/streams/%s?stream_type=all' % channel_id, channel_id,
            'Downloading stream JSON').get('stream')

        if not stream:
            raise ExtractorError('%s is offline' % channel_id, expected=True)

        # Channel name may be typed if different case than the original channel name
        # (e.g. http://www.twitch.tv/TWITCHPLAYSPOKEMON) that will lead to constructing
        # an invalid m3u8 URL. Working around by use of original channel name from stream
        # JSON and fallback to lowercase if it's not available.
        channel_id = stream.get('channel', {}).get('name') or channel_id.lower()

        access_token = self._call_api(
            'api/channels/%s/access_token' % channel_id, channel_id,
            'Downloading channel access token')

        query = {
            'allow_source': 'true',
            'allow_audio_only': 'true',
            'allow_spectre': 'true',
            'p': random.randint(1000000, 10000000),
            'player': 'twitchweb',
            'segment_preference': '4',
            'sig': access_token['sig'].encode('utf-8'),
            'token': access_token['token'].encode('utf-8'),
        }
        formats = self._extract_m3u8_formats(
            '%s/api/channel/hls/%s.m3u8?%s'
            % (self._USHER_BASE, channel_id, compat_urllib_parse_urlencode(query)),
            channel_id, 'mp4')
        self._prefer_source(formats)

        view_count = stream.get('viewers')
        timestamp = parse_iso8601(stream.get('created_at'))

        channel = stream['channel']
        title = self._live_title(channel.get('display_name') or channel.get('name'))
        description = channel.get('status')

        thumbnails = []
        for thumbnail_key, thumbnail_url in stream['preview'].items():
            m = re.search(r'(?P<width>\d+)x(?P<height>\d+)\.jpg$', thumbnail_key)
            if not m:
                continue
            thumbnails.append({
                'url': thumbnail_url,
                'width': int(m.group('width')),
                'height': int(m.group('height')),
            })

        return {
            'id': compat_str(stream['_id']),
            'display_id': channel_id,
            'title': title,
            'description': description,
            'thumbnails': thumbnails,
            'uploader': channel.get('display_name'),
            'uploader_id': channel.get('name'),
            'timestamp': timestamp,
            'view_count': view_count,
            'formats': formats,
            'is_live': True,
        }


class TwitchClipsIE(InfoExtractor):
    IE_NAME = 'twitch:clips'
    _VALID_URL = r'https?://clips\.twitch\.tv/(?:[^/]+/)*(?P<id>[^/?#&]+)'

    _TESTS = [{
        'url': 'https://clips.twitch.tv/ea/AggressiveCobraPoooound',
        'md5': '761769e1eafce0ffebfb4089cb3847cd',
        'info_dict': {
            'id': 'AggressiveCobraPoooound',
            'ext': 'mp4',
            'title': 'EA Play 2016 Live from the Novo Theatre',
            'thumbnail': r're:^https?://.*\.jpg',
            'creator': 'EA',
            'uploader': 'stereotype_',
            'uploader_id': 'stereotype_',
        },
    }, {
        # multiple formats
        'url': 'https://clips.twitch.tv/rflegendary/UninterestedBeeDAESuppy',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        clip = self._parse_json(
            self._search_regex(
                r'(?s)clipInfo\s*=\s*({.+?});', webpage, 'clip info'),
            video_id, transform_source=js_to_json)

        title = clip.get('channel_title') or self._og_search_title(webpage)

        formats = [{
            'url': option['source'],
            'format_id': option.get('quality'),
            'height': int_or_none(option.get('quality')),
        } for option in clip.get('quality_options', []) if option.get('source')]

        if not formats:
            formats = [{
                'url': clip['clip_video_url'],
            }]

        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'thumbnail': self._og_search_thumbnail(webpage),
            'creator': clip.get('broadcaster_display_name') or clip.get('broadcaster_login'),
            'uploader': clip.get('curator_login'),
            'uploader_id': clip.get('curator_display_name'),
            'formats': formats,
        }
Commit	Line	Data
04d02a9d	1	# coding: utf-8
ee1e1996 PH	2	from __future__ import unicode_literals
ee1e1996 PH	3
3182f3e2	4	import itertools
79e93125	5	import re
f353cbdb	6	import random
79e93125 PH	7
79e93125 PH	8	from .common import InfoExtractor
1cc79574	9	from ..compat import (
efe470e2	10	compat_HTTPError,
e704f87f	11	compat_parse_qs,
240b9b7a	12	compat_str,
15707c7e	13	compat_urllib_parse_urlencode,
e704f87f	14	compat_urllib_parse_urlparse,
03c635a4	15	compat_urlparse,
1cc79574 PH	16	)
1cc79574 PH	17	from ..utils import (
efe470e2	18	clean_html,
79e93125	19	ExtractorError,
7a6e8a1b	20	int_or_none,
778f9694	21	js_to_json,
8bbb4b56	22	orderedSet,
e704f87f	23	parse_duration,
355d074f	24	parse_iso8601,
264e77c4	25	update_url_query,
6e6bc8da	26	urlencode_postdata,
79e93125 PH	27	)
	28
	29
c5db6bb3	30	class TwitchBaseIE(InfoExtractor):
240b9b7a	31	_VALID_URL_BASE = r'https?://(?:www\.)?twitch\.tv'
c5db6bb3	32
46fd0dd5	33	_API_BASE = 'https://api.twitch.tv'
9f4576a7	34	_USHER_BASE = 'https://usher.ttvnw.net'
fbd9f6ea	35	_LOGIN_URL = 'http://www.twitch.tv/login'
95be29e1	36	_CLIENT_ID = 'jzkbprff40iqj646a697cyrvl0zt2m6'
499bfcbf	37	_NETRC_MACHINE = 'twitch'
79e93125	38
355d074f S	39	def _handle_error(self, response):
	40	if not isinstance(response, dict):
	41	return
	42	error = response.get('error')
	43	if error:
	44	raise ExtractorError(
	45	'%s returned error: %s - %s' % (self.IE_NAME, error, response.get('message')),
	46	expected=True)
	47
e3f6b569	48	def _call_api(self, path, item_id, note):
e3f6b569	49	response = self._download_json(
95be29e1 S	50	'%s/%s' % (self._API_BASE, path), item_id, note,
95be29e1 S	51	headers={'Client-ID': self._CLIENT_ID})
355d074f S	52	self._handle_error(response)
	53	return response
	54
c5db6bb3 S	55	def _real_initialize(self):
	56	self._login()
	57
	58	def _login(self):
	59	(username, password) = self._get_login_info()
	60	if username is None:
	61	return
	62
efe470e2 S	63	def fail(message):
	64	raise ExtractorError(
	65	'Unable to login. Twitch said: %s' % message, expected=True)
	66
fbd9f6ea	67	login_page, handle = self._download_webpage_handle(
c5db6bb3 S	68	self._LOGIN_URL, None, 'Downloading login page')
c5db6bb3 S	69
efe470e2 S	70	# Some TOR nodes and public proxies are blocked completely
	71	if 'blacklist_message' in login_page:
	72	fail(clean_html(login_page))
	73
f8da79f8	74	login_form = self._hidden_inputs(login_page)
c5db6bb3	75
9296e92e	76	login_form.update({
fbd9f6ea S	77	'username': username,
fbd9f6ea S	78	'password': password,
9296e92e	79	})
c5db6bb3	80
fbd9f6ea S	81	redirect_url = handle.geturl()
fbd9f6ea S	82
03c635a4 S	83	post_url = self._search_regex(
03c635a4 S	84	r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page,
fbd9f6ea	85	'post url', default=redirect_url, group='url')
03c635a4 S	86
03c635a4 S	87	if not post_url.startswith('http'):
fbd9f6ea	88	post_url = compat_urlparse.urljoin(redirect_url, post_url)
03c635a4	89
efe470e2	90	headers = {'Referer': redirect_url}
c5db6bb3	91
efe470e2 S	92	try:
	93	response = self._download_json(
	94	post_url, None, 'Logging in as %s' % username,
	95	data=urlencode_postdata(login_form),
	96	headers=headers)
	97	except ExtractorError as e:
	98	if isinstance(e.cause, compat_HTTPError) and e.cause.code == 400:
	99	response = self._parse_json(
	100	e.cause.read().decode('utf-8'), None)
	101	fail(response['message'])
	102	raise
	103
	104	if response.get('redirect'):
	105	self._download_webpage(
	106	response['redirect'], None, 'Downloading login redirect page',
	107	headers=headers)
17b41a33	108
d0e958c7 PH	109	def _prefer_source(self, formats):
	110	try:
	111	source = next(f for f in formats if f['format_id'] == 'Source')
	112	source['preference'] = 10
	113	except StopIteration:
	114	pass # No Source stream present
	115	self._sort_formats(formats)
	116
c5db6bb3 S	117
	118	class TwitchItemBaseIE(TwitchBaseIE):
	119	def _download_info(self, item, item_id):
e3f6b569 S	120	return self._extract_info(self._call_api(
e3f6b569 S	121	'kraken/videos/%s%s' % (item, item_id), item_id,
c5db6bb3	122	'Downloading %s info JSON' % self._ITEM_TYPE))
a22524b0	123
c5db6bb3 S	124	def _extract_media(self, item_id):
c5db6bb3 S	125	info = self._download_info(self._ITEM_SHORTCUT, item_id)
e3f6b569 S	126	response = self._call_api(
e3f6b569 S	127	'api/videos/%s%s' % (self._ITEM_SHORTCUT, item_id), item_id,
c5db6bb3	128	'Downloading %s playlist JSON' % self._ITEM_TYPE)
355d074f S	129	entries = []
	130	chunks = response['chunks']
	131	qualities = list(chunks.keys())
	132	for num, fragment in enumerate(zip(*chunks.values()), start=1):
	133	formats = []
	134	for fmt_num, fragment_fmt in enumerate(fragment):
	135	format_id = qualities[fmt_num]
	136	fmt = {
	137	'url': fragment_fmt['url'],
	138	'format_id': format_id,
	139	'quality': 1 if format_id == 'live' else 0,
	140	}
	141	m = re.search(r'^(?P<height>\d+)[Pp]', format_id)
	142	if m:
	143	fmt['height'] = int(m.group('height'))
	144	formats.append(fmt)
	145	self._sort_formats(formats)
	146	entry = dict(info)
159444a6	147	entry['id'] = '%s_%d' % (entry['id'], num)
355d074f S	148	entry['title'] = '%s part %d' % (entry['title'], num)
	149	entry['formats'] = formats
	150	entries.append(entry)
013bfdd8	151	return self.playlist_result(entries, info['id'], info['title'])
355d074f S	152
	153	def _extract_info(self, info):
	154	return {
	155	'id': info['_id'],
369c12e0	156	'title': info.get('title') or 'Untitled Broadcast',
7a6e8a1b S	157	'description': info.get('description'),
	158	'duration': int_or_none(info.get('length')),
	159	'thumbnail': info.get('preview'),
	160	'uploader': info.get('channel', {}).get('display_name'),
	161	'uploader_id': info.get('channel', {}).get('name'),
	162	'timestamp': parse_iso8601(info.get('recorded_at')),
	163	'view_count': int_or_none(info.get('views')),
355d074f S	164	}
355d074f S	165
c5db6bb3 S	166	def _real_extract(self, url):
c5db6bb3 S	167	return self._extract_media(self._match_id(url))
04d02a9d	168
04d02a9d	169
c5db6bb3 S	170	class TwitchVideoIE(TwitchItemBaseIE):
c5db6bb3 S	171	IE_NAME = 'twitch:video'
93f78707	172	_VALID_URL = r'%s/[^/]+/b/(?P<id>\d+)' % TwitchBaseIE._VALID_URL_BASE
c5db6bb3 S	173	_ITEM_TYPE = 'video'
c5db6bb3 S	174	_ITEM_SHORTCUT = 'a'
04d02a9d	175
c5db6bb3 S	176	_TEST = {
	177	'url': 'http://www.twitch.tv/riotgames/b/577357806',
	178	'info_dict': {
	179	'id': 'a577357806',
	180	'title': 'Worlds Semifinals - Star Horn Royal Club vs. OMG',
	181	},
	182	'playlist_mincount': 12,
0db3a661	183	'skip': 'HTTP Error 404: Not Found',
c5db6bb3	184	}
04d02a9d	185
04d02a9d	186
c5db6bb3 S	187	class TwitchChapterIE(TwitchItemBaseIE):
c5db6bb3 S	188	IE_NAME = 'twitch:chapter'
93f78707	189	_VALID_URL = r'%s/[^/]+/c/(?P<id>\d+)' % TwitchBaseIE._VALID_URL_BASE
c5db6bb3 S	190	_ITEM_TYPE = 'chapter'
c5db6bb3 S	191	_ITEM_SHORTCUT = 'c'
04d02a9d	192
78111136	193	_TESTS = [{
c5db6bb3 S	194	'url': 'http://www.twitch.tv/acracingleague/c/5285812',
	195	'info_dict': {
	196	'id': 'c5285812',
	197	'title': 'ACRL Off Season - Sports Cars @ Nordschleife',
	198	},
	199	'playlist_mincount': 3,
0db3a661	200	'skip': 'HTTP Error 404: Not Found',
78111136 PH	201	}, {
	202	'url': 'http://www.twitch.tv/tsm_theoddone/c/2349361',
	203	'only_matching': True,
	204	}]
c5db6bb3 S	205
	206
	207	class TwitchVodIE(TwitchItemBaseIE):
	208	IE_NAME = 'twitch:vod'
3f1ce168 S	209	_VALID_URL = r'''(?x)
	210	https?://
	211	(?:
0b23c222	212	(?:www\.)?twitch\.tv/(?:[^/]+/v\|videos)/\|
3f1ce168 S	213	player\.twitch\.tv/\?.*?\bvideo=v
	214	)
	215	(?P<id>\d+)
	216	'''
c5db6bb3 S	217	_ITEM_TYPE = 'vod'
	218	_ITEM_SHORTCUT = 'v'
	219
9c724a98	220	_TESTS = [{
e704f87f	221	'url': 'http://www.twitch.tv/riotgames/v/6528877?t=5m10s',
c5db6bb3	222	'info_dict': {
ac0474f8	223	'id': 'v6528877',
c5db6bb3	224	'ext': 'mp4',
ac0474f8	225	'title': 'LCK Summer Split - Week 6 Day 1',
ec85ded8	226	'thumbnail': r're:^https?://.*\.jpg$',
ac0474f8 YCH	227	'duration': 17208,
	228	'timestamp': 1435131709,
	229	'upload_date': '20150624',
	230	'uploader': 'Riot Games',
	231	'uploader_id': 'riotgames',
c5db6bb3	232	'view_count': int,
e704f87f	233	'start_time': 310,
c5db6bb3 S	234	},
	235	'params': {
	236	# m3u8 download
	237	'skip_download': True,
	238	},
9c724a98 S	239	}, {
	240	# Untitled broadcast (title is None)
	241	'url': 'http://www.twitch.tv/belkao_o/v/11230755',
	242	'info_dict': {
	243	'id': 'v11230755',
	244	'ext': 'mp4',
	245	'title': 'Untitled Broadcast',
ec85ded8	246	'thumbnail': r're:^https?://.*\.jpg$',
9c724a98 S	247	'duration': 1638,
	248	'timestamp': 1439746708,
	249	'upload_date': '20150816',
	250	'uploader': 'BelkAO_o',
	251	'uploader_id': 'belkao_o',
	252	'view_count': int,
	253	},
	254	'params': {
	255	# m3u8 download
	256	'skip_download': True,
	257	},
9bd7bd0b	258	'skip': 'HTTP Error 404: Not Found',
3f1ce168 S	259	}, {
	260	'url': 'http://player.twitch.tv/?t=5m10s&video=v6528877',
	261	'only_matching': True,
0b23c222 S	262	}, {
	263	'url': 'https://www.twitch.tv/videos/6528877',
	264	'only_matching': True,
9c724a98	265	}]
04d02a9d	266
79e93125	267	def _real_extract(self, url):
c5db6bb3	268	item_id = self._match_id(url)
e5e99661	269
c5db6bb3	270	info = self._download_info(self._ITEM_SHORTCUT, item_id)
e3f6b569 S	271	access_token = self._call_api(
e3f6b569 S	272	'api/vods/%s/access_token' % item_id, item_id,
350c9481	273	'Downloading %s access token' % self._ITEM_TYPE)
e5e99661	274
c5db6bb3	275	formats = self._extract_m3u8_formats(
350c9481 S	276	'%s/vod/%s?%s' % (
350c9481 S	277	self._USHER_BASE, item_id,
15707c7e	278	compat_urllib_parse_urlencode({
e5e99661	279	'allow_source': 'true',
ac455055	280	'allow_audio_only': 'true',
e5e99661 S	281	'allow_spectre': 'true',
	282	'player': 'twitchweb',
	283	'nauth': access_token['token'],
	284	'nauthsig': access_token['sig'],
	285	})),
631d4c87	286	item_id, 'mp4', entry_protocol='m3u8_native')
e5e99661	287
d0e958c7	288	self._prefer_source(formats)
c5db6bb3	289	info['formats'] = formats
e704f87f NH	290
	291	parsed_url = compat_urllib_parse_urlparse(url)
	292	query = compat_parse_qs(parsed_url.query)
	293	if 't' in query:
	294	info['start_time'] = parse_duration(query['t'][0])
	295
264e77c4 S	296	if info.get('timestamp') is not None:
	297	info['subtitles'] = {
	298	'rechat': [{
	299	'url': update_url_query(
	300	'https://rechat.twitch.tv/rechat-messages', {
	301	'video_id': 'v%s' % item_id,
	302	'start': info['timestamp'],
	303	}),
	304	'ext': 'json',
	305	}],
	306	}
	307
c5db6bb3 S	308	return info
	309
	310
	311	class TwitchPlaylistBaseIE(TwitchBaseIE):
e3f6b569	312	_PLAYLIST_PATH = 'kraken/channels/%s/videos/?offset=%d&limit=%d'
ececca6c	313	_PAGE_LIMIT = 100
c5db6bb3 S	314
c5db6bb3 S	315	def _extract_playlist(self, channel_id):
e3f6b569 S	316	info = self._call_api(
e3f6b569 S	317	'kraken/channels/%s' % channel_id,
c5db6bb3 S	318	channel_id, 'Downloading channel info JSON')
	319	channel_name = info.get('display_name') or info.get('name')
	320	entries = []
	321	offset = 0
	322	limit = self._PAGE_LIMIT
539a1641 S	323	broken_paging_detected = False
539a1641 S	324	counter_override = None
c5db6bb3	325	for counter in itertools.count(1):
e3f6b569 S	326	response = self._call_api(
e3f6b569 S	327	self._PLAYLIST_PATH % (channel_id, offset, limit),
539a1641	328	channel_id,
93753aad	329	'Downloading %s JSON page %s'
539a1641	330	% (self._PLAYLIST_TYPE, counter_override or counter))
c2e64f71 S	331	page_entries = self._extract_playlist_page(response)
c2e64f71 S	332	if not page_entries:
c5db6bb3	333	break
539a1641 S	334	total = int_or_none(response.get('_total'))
	335	# Since the beginning of March 2016 twitch's paging mechanism
	336	# is completely broken on the twitch side. It simply ignores
	337	# a limit and returns the whole offset number of videos.
	338	# Working around by just requesting all videos at once.
dec2cae0	339	# Upd: pagination bug was fixed by twitch on 15.03.2016.
539a1641 S	340	if not broken_paging_detected and total and len(page_entries) > limit:
539a1641 S	341	self.report_warning(
dec2cae0	342	'Twitch pagination is broken on twitch side, requesting all videos at once',
539a1641 S	343	channel_id)
539a1641 S	344	broken_paging_detected = True
a8276b26	345	offset = total
539a1641 S	346	counter_override = '(all at once)'
539a1641 S	347	continue
c2e64f71	348	entries.extend(page_entries)
539a1641 S	349	if broken_paging_detected or total and len(page_entries) >= total:
539a1641 S	350	break
c5db6bb3	351	offset += limit
c2e64f71	352	return self.playlist_result(
8bbb4b56	353	[self.url_result(entry) for entry in orderedSet(entries)],
c2e64f71 S	354	channel_id, channel_name)
	355
	356	def _extract_playlist_page(self, response):
	357	videos = response.get('videos')
	358	return [video['url'] for video in videos] if videos else []
c5db6bb3 S	359
	360	def _real_extract(self, url):
	361	return self._extract_playlist(self._match_id(url))
	362
	363
	364	class TwitchProfileIE(TwitchPlaylistBaseIE):
	365	IE_NAME = 'twitch:profile'
	366	_VALID_URL = r'%s/(?P<id>[^/]+)/profile/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
	367	_PLAYLIST_TYPE = 'profile'
	368
	369	_TEST = {
	370	'url': 'http://www.twitch.tv/vanillatv/profile',
	371	'info_dict': {
	372	'id': 'vanillatv',
	373	'title': 'VanillaTV',
	374	},
	375	'playlist_mincount': 412,
	376	}
	377
	378
93753aad S	379	class TwitchVideosBaseIE(TwitchPlaylistBaseIE):
	380	_VALID_URL_VIDEOS_BASE = r'%s/(?P<id>[^/]+)/videos' % TwitchBaseIE._VALID_URL_BASE
	381	_PLAYLIST_PATH = TwitchPlaylistBaseIE._PLAYLIST_PATH + '&broadcast_type='
	382
	383
	384	class TwitchAllVideosIE(TwitchVideosBaseIE):
	385	IE_NAME = 'twitch:videos:all'
	386	_VALID_URL = r'%s/all' % TwitchVideosBaseIE._VALID_URL_VIDEOS_BASE
	387	_PLAYLIST_PATH = TwitchVideosBaseIE._PLAYLIST_PATH + 'archive,upload,highlight'
	388	_PLAYLIST_TYPE = 'all videos'
	389
	390	_TEST = {
	391	'url': 'https://www.twitch.tv/spamfish/videos/all',
	392	'info_dict': {
	393	'id': 'spamfish',
	394	'title': 'Spamfish',
	395	},
	396	'playlist_mincount': 869,
	397	}
	398
	399
	400	class TwitchUploadsIE(TwitchVideosBaseIE):
	401	IE_NAME = 'twitch:videos:uploads'
	402	_VALID_URL = r'%s/uploads' % TwitchVideosBaseIE._VALID_URL_VIDEOS_BASE
	403	_PLAYLIST_PATH = TwitchVideosBaseIE._PLAYLIST_PATH + 'upload'
	404	_PLAYLIST_TYPE = 'uploads'
	405
	406	_TEST = {
	407	'url': 'https://www.twitch.tv/spamfish/videos/uploads',
	408	'info_dict': {
	409	'id': 'spamfish',
	410	'title': 'Spamfish',
	411	},
	412	'playlist_mincount': 0,
	413	}
	414
	415
	416	class TwitchPastBroadcastsIE(TwitchVideosBaseIE):
	417	IE_NAME = 'twitch:videos:past-broadcasts'
	418	_VALID_URL = r'%s/past-broadcasts' % TwitchVideosBaseIE._VALID_URL_VIDEOS_BASE
	419	_PLAYLIST_PATH = TwitchVideosBaseIE._PLAYLIST_PATH + 'archive'
c5db6bb3 S	420	_PLAYLIST_TYPE = 'past broadcasts'
	421
	422	_TEST = {
93753aad S	423	'url': 'https://www.twitch.tv/spamfish/videos/past-broadcasts',
	424	'info_dict': {
	425	'id': 'spamfish',
	426	'title': 'Spamfish',
	427	},
	428	'playlist_mincount': 0,
	429	}
	430
	431
	432	class TwitchHighlightsIE(TwitchVideosBaseIE):
	433	IE_NAME = 'twitch:videos:highlights'
	434	_VALID_URL = r'%s/highlights' % TwitchVideosBaseIE._VALID_URL_VIDEOS_BASE
	435	_PLAYLIST_PATH = TwitchVideosBaseIE._PLAYLIST_PATH + 'highlight'
	436	_PLAYLIST_TYPE = 'highlights'
	437
	438	_TEST = {
	439	'url': 'https://www.twitch.tv/spamfish/videos/highlights',
c5db6bb3 S	440	'info_dict': {
	441	'id': 'spamfish',
	442	'title': 'Spamfish',
	443	},
93753aad	444	'playlist_mincount': 805,
c5db6bb3	445	}
240b9b7a S	446
	447
	448	class TwitchStreamIE(TwitchBaseIE):
	449	IE_NAME = 'twitch:stream'
9db8f6c5 S	450	_VALID_URL = r'''(?x)
	451	https?://
	452	(?:
	453	(?:www\.)?twitch\.tv/\|
	454	player\.twitch\.tv/\?.*?\bchannel=
	455	)
	456	(?P<id>[^/#?]+)
	457	'''
240b9b7a	458
faa1f83a	459	_TESTS = [{
240b9b7a S	460	'url': 'http://www.twitch.tv/shroomztv',
	461	'info_dict': {
	462	'id': '12772022048',
	463	'display_id': 'shroomztv',
	464	'ext': 'mp4',
	465	'title': 're:^ShroomzTV [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
	466	'description': 'H1Z1 - lonewolfing with ShroomzTV \| A3 Battle Royale later - @ShroomzTV',
	467	'is_live': True,
	468	'timestamp': 1421928037,
	469	'upload_date': '20150122',
	470	'uploader': 'ShroomzTV',
	471	'uploader_id': 'shroomztv',
	472	'view_count': int,
	473	},
	474	'params': {
	475	# m3u8 download
	476	'skip_download': True,
	477	},
faa1f83a S	478	}, {
	479	'url': 'http://www.twitch.tv/miracle_doto#profile-0',
	480	'only_matching': True,
9db8f6c5 S	481	}, {
	482	'url': 'https://player.twitch.tv/?channel=lotsofs',
	483	'only_matching': True,
faa1f83a	484	}]
240b9b7a	485
9db8f6c5 S	486	@classmethod
	487	def suitable(cls, url):
	488	return (False
	489	if any(ie.suitable(url) for ie in (
	490	TwitchVideoIE,
	491	TwitchChapterIE,
	492	TwitchVodIE,
	493	TwitchProfileIE,
	494	TwitchAllVideosIE,
	495	TwitchUploadsIE,
	496	TwitchPastBroadcastsIE,
	497	TwitchHighlightsIE))
	498	else super(TwitchStreamIE, cls).suitable(url))
	499
240b9b7a S	500	def _real_extract(self, url):
	501	channel_id = self._match_id(url)
	502
e3f6b569	503	stream = self._call_api(
9aa929d3	504	'kraken/streams/%s?stream_type=all' % channel_id, channel_id,
240b9b7a S	505	'Downloading stream JSON').get('stream')
240b9b7a S	506
240b9b7a	507	if not stream:
cc764a6d	508	raise ExtractorError('%s is offline' % channel_id, expected=True)
240b9b7a	509
06966677 S	510	# Channel name may be typed if different case than the original channel name
	511	# (e.g. http://www.twitch.tv/TWITCHPLAYSPOKEMON) that will lead to constructing
	512	# an invalid m3u8 URL. Working around by use of original channel name from stream
	513	# JSON and fallback to lowercase if it's not available.
1793d71d S	514	channel_id = stream.get('channel', {}).get('name') or channel_id.lower()
1793d71d S	515
e3f6b569 S	516	access_token = self._call_api(
e3f6b569 S	517	'api/channels/%s/access_token' % channel_id, channel_id,
240b9b7a S	518	'Downloading channel access token')
	519
	520	query = {
	521	'allow_source': 'true',
ac455055	522	'allow_audio_only': 'true',
9aa929d3	523	'allow_spectre': 'true',
f353cbdb	524	'p': random.randint(1000000, 10000000),
240b9b7a S	525	'player': 'twitchweb',
240b9b7a S	526	'segment_preference': '4',
d34e7949 S	527	'sig': access_token['sig'].encode('utf-8'),
d34e7949 S	528	'token': access_token['token'].encode('utf-8'),
240b9b7a	529	}
240b9b7a S	530	formats = self._extract_m3u8_formats(
240b9b7a S	531	'%s/api/channel/hls/%s.m3u8?%s'
15707c7e	532	% (self._USHER_BASE, channel_id, compat_urllib_parse_urlencode(query)),
240b9b7a	533	channel_id, 'mp4')
d0e958c7	534	self._prefer_source(formats)
240b9b7a S	535
	536	view_count = stream.get('viewers')
	537	timestamp = parse_iso8601(stream.get('created_at'))
	538
	539	channel = stream['channel']
	540	title = self._live_title(channel.get('display_name') or channel.get('name'))
	541	description = channel.get('status')
	542
	543	thumbnails = []
	544	for thumbnail_key, thumbnail_url in stream['preview'].items():
	545	m = re.search(r'(?P<width>\d+)x(?P<height>\d+)\.jpg$', thumbnail_key)
	546	if not m:
	547	continue
	548	thumbnails.append({
	549	'url': thumbnail_url,
	550	'width': int(m.group('width')),
	551	'height': int(m.group('height')),
	552	})
	553
	554	return {
	555	'id': compat_str(stream['_id']),
	556	'display_id': channel_id,
	557	'title': title,
	558	'description': description,
	559	'thumbnails': thumbnails,
	560	'uploader': channel.get('display_name'),
	561	'uploader_id': channel.get('name'),
	562	'timestamp': timestamp,
	563	'view_count': view_count,
	564	'formats': formats,
	565	'is_live': True,
12d1fb5a	566	}
778f9694 S	567
	568
	569	class TwitchClipsIE(InfoExtractor):
	570	IE_NAME = 'twitch:clips'
	571	_VALID_URL = r'https?://clips\.twitch\.tv/(?:[^/]+/)*(?P<id>[^/?#&]+)'
	572
74ba450a	573	_TESTS = [{
778f9694 S	574	'url': 'https://clips.twitch.tv/ea/AggressiveCobraPoooound',
	575	'md5': '761769e1eafce0ffebfb4089cb3847cd',
	576	'info_dict': {
	577	'id': 'AggressiveCobraPoooound',
	578	'ext': 'mp4',
	579	'title': 'EA Play 2016 Live from the Novo Theatre',
ec85ded8	580	'thumbnail': r're:^https?://.*\.jpg',
778f9694 S	581	'creator': 'EA',
	582	'uploader': 'stereotype_',
	583	'uploader_id': 'stereotype_',
	584	},
74ba450a S	585	}, {
	586	# multiple formats
	587	'url': 'https://clips.twitch.tv/rflegendary/UninterestedBeeDAESuppy',
	588	'only_matching': True,
	589	}]
778f9694 S	590
	591	def _real_extract(self, url):
	592	video_id = self._match_id(url)
	593
	594	webpage = self._download_webpage(url, video_id)
	595
	596	clip = self._parse_json(
	597	self._search_regex(
	598	r'(?s)clipInfo\s=\s({.+?});', webpage, 'clip info'),
	599	video_id, transform_source=js_to_json)
	600
74ba450a S	601	title = clip.get('channel_title') or self._og_search_title(webpage)
	602
	603	formats = [{
	604	'url': option['source'],
	605	'format_id': option.get('quality'),
	606	'height': int_or_none(option.get('quality')),
	607	} for option in clip.get('quality_options', []) if option.get('source')]
	608
	609	if not formats:
	610	formats = [{
	611	'url': clip['clip_video_url'],
	612	}]
778f9694	613
0cacae28 S	614	self._sort_formats(formats)
0cacae28 S	615
778f9694 S	616	return {
778f9694 S	617	'id': video_id,
778f9694 S	618	'title': title,
	619	'thumbnail': self._og_search_thumbnail(webpage),
	620	'creator': clip.get('broadcaster_display_name') or clip.get('broadcaster_login'),
	621	'uploader': clip.get('curator_login'),
	622	'uploader_id': clip.get('curator_display_name'),
74ba450a	623	'formats': formats,
778f9694	624	}