[yt-dlp.git] / youtube_dl / extractor / twitch.py

# coding: utf-8
from __future__ import unicode_literals

import itertools
import re
import random

from .common import InfoExtractor
from ..compat import (
    compat_str,
    compat_urllib_parse,
    compat_urllib_request,
)
from ..utils import (
    ExtractorError,
    parse_iso8601,
)


class TwitchBaseIE(InfoExtractor):
    _VALID_URL_BASE = r'https?://(?:www\.)?twitch\.tv'

    _API_BASE = 'https://api.twitch.tv'
    _USHER_BASE = 'http://usher.twitch.tv'
    _LOGIN_URL = 'https://secure.twitch.tv/user/login'
    _NETRC_MACHINE = 'twitch'

    def _handle_error(self, response):
        if not isinstance(response, dict):
            return
        error = response.get('error')
        if error:
            raise ExtractorError(
                '%s returned error: %s - %s' % (self.IE_NAME, error, response.get('message')),
                expected=True)

    def _download_json(self, url, video_id, note='Downloading JSON metadata'):
        headers = {
            'Referer': 'http://api.twitch.tv/crossdomain/receiver.html?v=2',
            'X-Requested-With': 'XMLHttpRequest',
        }
        for cookie in self._downloader.cookiejar:
            if cookie.name == 'api_token':
                headers['Twitch-Api-Token'] = cookie.value
        request = compat_urllib_request.Request(url, headers=headers)
        response = super(TwitchBaseIE, self)._download_json(request, video_id, note)
        self._handle_error(response)
        return response

    def _real_initialize(self):
        self._login()

    def _login(self):
        (username, password) = self._get_login_info()
        if username is None:
            return

        login_page = self._download_webpage(
            self._LOGIN_URL, None, 'Downloading login page')

        authenticity_token = self._search_regex(
            r'<input name="authenticity_token" type="hidden" value="([^"]+)"',
            login_page, 'authenticity token')

        login_form = {
            'utf8': '✓'.encode('utf-8'),
            'authenticity_token': authenticity_token,
            'redirect_on_login': '',
            'embed_form': 'false',
            'mp_source_action': '',
            'follow': '',
            'user[login]': username,
            'user[password]': password,
        }

        request = compat_urllib_request.Request(
            self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
        request.add_header('Referer', self._LOGIN_URL)
        response = self._download_webpage(
            request, None, 'Logging in as %s' % username)

        m = re.search(
            r"id=([\"'])login_error_message\1[^>]*>(?P<msg>[^<]+)", response)
        if m:
            raise ExtractorError(
                'Unable to login: %s' % m.group('msg').strip(), expected=True)

    def _prefer_source(self, formats):
        try:
            source = next(f for f in formats if f['format_id'] == 'Source')
            source['preference'] = 10
        except StopIteration:
            pass  # No Source stream present
        self._sort_formats(formats)


class TwitchItemBaseIE(TwitchBaseIE):
    def _download_info(self, item, item_id):
        return self._extract_info(self._download_json(
            '%s/kraken/videos/%s%s' % (self._API_BASE, item, item_id), item_id,
            'Downloading %s info JSON' % self._ITEM_TYPE))

    def _extract_media(self, item_id):
        info = self._download_info(self._ITEM_SHORTCUT, item_id)
        response = self._download_json(
            '%s/api/videos/%s%s' % (self._API_BASE, self._ITEM_SHORTCUT, item_id), item_id,
            'Downloading %s playlist JSON' % self._ITEM_TYPE)
        entries = []
        chunks = response['chunks']
        qualities = list(chunks.keys())
        for num, fragment in enumerate(zip(*chunks.values()), start=1):
            formats = []
            for fmt_num, fragment_fmt in enumerate(fragment):
                format_id = qualities[fmt_num]
                fmt = {
                    'url': fragment_fmt['url'],
                    'format_id': format_id,
                    'quality': 1 if format_id == 'live' else 0,
                }
                m = re.search(r'^(?P<height>\d+)[Pp]', format_id)
                if m:
                    fmt['height'] = int(m.group('height'))
                formats.append(fmt)
            self._sort_formats(formats)
            entry = dict(info)
            entry['id'] = '%s_%d' % (entry['id'], num)
            entry['title'] = '%s part %d' % (entry['title'], num)
            entry['formats'] = formats
            entries.append(entry)
        return self.playlist_result(entries, info['id'], info['title'])

    def _extract_info(self, info):
        return {
            'id': info['_id'],
            'title': info['title'],
            'description': info['description'],
            'duration': info['length'],
            'thumbnail': info['preview'],
            'uploader': info['channel']['display_name'],
            'uploader_id': info['channel']['name'],
            'timestamp': parse_iso8601(info['recorded_at']),
            'view_count': info['views'],
        }

    def _real_extract(self, url):
        return self._extract_media(self._match_id(url))


class TwitchVideoIE(TwitchItemBaseIE):
    IE_NAME = 'twitch:video'
    _VALID_URL = r'%s/[^/]+/b/(?P<id>[^/]+)' % TwitchBaseIE._VALID_URL_BASE
    _ITEM_TYPE = 'video'
    _ITEM_SHORTCUT = 'a'

    _TEST = {
        'url': 'http://www.twitch.tv/riotgames/b/577357806',
        'info_dict': {
            'id': 'a577357806',
            'title': 'Worlds Semifinals - Star Horn Royal Club vs. OMG',
        },
        'playlist_mincount': 12,
    }


class TwitchChapterIE(TwitchItemBaseIE):
    IE_NAME = 'twitch:chapter'
    _VALID_URL = r'%s/[^/]+/c/(?P<id>[^/]+)' % TwitchBaseIE._VALID_URL_BASE
    _ITEM_TYPE = 'chapter'
    _ITEM_SHORTCUT = 'c'

    _TESTS = [{
        'url': 'http://www.twitch.tv/acracingleague/c/5285812',
        'info_dict': {
            'id': 'c5285812',
            'title': 'ACRL Off Season - Sports Cars @ Nordschleife',
        },
        'playlist_mincount': 3,
    }, {
        'url': 'http://www.twitch.tv/tsm_theoddone/c/2349361',
        'only_matching': True,
    }]


class TwitchVodIE(TwitchItemBaseIE):
    IE_NAME = 'twitch:vod'
    _VALID_URL = r'%s/[^/]+/v/(?P<id>[^/]+)' % TwitchBaseIE._VALID_URL_BASE
    _ITEM_TYPE = 'vod'
    _ITEM_SHORTCUT = 'v'

    _TEST = {
        'url': 'http://www.twitch.tv/ksptv/v/3622000',
        'info_dict': {
            'id': 'v3622000',
            'ext': 'mp4',
            'title': '''KSPTV: Squadcast: "Everyone's on vacation so here's Dahud" Edition!''',
            'thumbnail': 're:^https?://.*\.jpg$',
            'duration': 6951,
            'timestamp': 1419028564,
            'upload_date': '20141219',
            'uploader': 'KSPTV',
            'uploader_id': 'ksptv',
            'view_count': int,
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }

    def _real_extract(self, url):
        item_id = self._match_id(url)
        info = self._download_info(self._ITEM_SHORTCUT, item_id)
        access_token = self._download_json(
            '%s/api/vods/%s/access_token' % (self._API_BASE, item_id), item_id,
            'Downloading %s access token' % self._ITEM_TYPE)
        formats = self._extract_m3u8_formats(
            '%s/vod/%s?nauth=%s&nauthsig=%s'
            % (self._USHER_BASE, item_id, access_token['token'], access_token['sig']),
            item_id, 'mp4')
        self._prefer_source(formats)
        info['formats'] = formats
        return info


class TwitchPlaylistBaseIE(TwitchBaseIE):
    _PLAYLIST_URL = '%s/kraken/channels/%%s/videos/?offset=%%d&limit=%%d' % TwitchBaseIE._API_BASE
    _PAGE_LIMIT = 100

    def _extract_playlist(self, channel_id):
        info = self._download_json(
            '%s/kraken/channels/%s' % (self._API_BASE, channel_id),
            channel_id, 'Downloading channel info JSON')
        channel_name = info.get('display_name') or info.get('name')
        entries = []
        offset = 0
        limit = self._PAGE_LIMIT
        for counter in itertools.count(1):
            response = self._download_json(
                self._PLAYLIST_URL % (channel_id, offset, limit),
                channel_id, 'Downloading %s videos JSON page %d' % (self._PLAYLIST_TYPE, counter))
            page_entries = self._extract_playlist_page(response)
            if not page_entries:
                break
            entries.extend(page_entries)
            offset += limit
        return self.playlist_result(
            [self.url_result(entry) for entry in set(entries)],
            channel_id, channel_name)

    def _extract_playlist_page(self, response):
        videos = response.get('videos')
        return [video['url'] for video in videos] if videos else []

    def _real_extract(self, url):
        return self._extract_playlist(self._match_id(url))


class TwitchProfileIE(TwitchPlaylistBaseIE):
    IE_NAME = 'twitch:profile'
    _VALID_URL = r'%s/(?P<id>[^/]+)/profile/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
    _PLAYLIST_TYPE = 'profile'

    _TEST = {
        'url': 'http://www.twitch.tv/vanillatv/profile',
        'info_dict': {
            'id': 'vanillatv',
            'title': 'VanillaTV',
        },
        'playlist_mincount': 412,
    }


class TwitchPastBroadcastsIE(TwitchPlaylistBaseIE):
    IE_NAME = 'twitch:past_broadcasts'
    _VALID_URL = r'%s/(?P<id>[^/]+)/profile/past_broadcasts/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
    _PLAYLIST_URL = TwitchPlaylistBaseIE._PLAYLIST_URL + '&broadcasts=true'
    _PLAYLIST_TYPE = 'past broadcasts'

    _TEST = {
        'url': 'http://www.twitch.tv/spamfish/profile/past_broadcasts',
        'info_dict': {
            'id': 'spamfish',
            'title': 'Spamfish',
        },
        'playlist_mincount': 54,
    }


class TwitchBookmarksIE(TwitchPlaylistBaseIE):
    IE_NAME = 'twitch:bookmarks'
    _VALID_URL = r'%s/(?P<id>[^/]+)/profile/bookmarks/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
    _PLAYLIST_URL = '%s/api/bookmark/?user=%%s&offset=%%d&limit=%%d' % TwitchBaseIE._API_BASE
    _PLAYLIST_TYPE = 'bookmarks'

    _TEST = {
        'url': 'http://www.twitch.tv/ognos/profile/bookmarks',
        'info_dict': {
            'id': 'ognos',
            'title': 'Ognos',
        },
        'playlist_mincount': 3,
    }

    def _extract_playlist_page(self, response):
        entries = []
        for bookmark in response.get('bookmarks', []):
            video = bookmark.get('video')
            if not video:
                continue
            entries.append(video['url'])
        return entries


class TwitchStreamIE(TwitchBaseIE):
    IE_NAME = 'twitch:stream'
    _VALID_URL = r'%s/(?P<id>[^/]+)/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE

    _TEST = {
        'url': 'http://www.twitch.tv/shroomztv',
        'info_dict': {
            'id': '12772022048',
            'display_id': 'shroomztv',
            'ext': 'mp4',
            'title': 're:^ShroomzTV [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
            'description': 'H1Z1 - lonewolfing with ShroomzTV | A3 Battle Royale later - @ShroomzTV',
            'is_live': True,
            'timestamp': 1421928037,
            'upload_date': '20150122',
            'uploader': 'ShroomzTV',
            'uploader_id': 'shroomztv',
            'view_count': int,
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }

    def _real_extract(self, url):
        channel_id = self._match_id(url)

        stream = self._download_json(
            '%s/kraken/streams/%s' % (self._API_BASE, channel_id), channel_id,
            'Downloading stream JSON').get('stream')

        # Fallback on profile extraction if stream is offline
        if not stream:
            return self.url_result(
                'http://www.twitch.tv/%s/profile' % channel_id,
                'TwitchProfile', channel_id)

        access_token = self._download_json(
            '%s/api/channels/%s/access_token' % (self._API_BASE, channel_id), channel_id,
            'Downloading channel access token')

        query = {
            'allow_source': 'true',
            'p': random.randint(1000000, 10000000),
            'player': 'twitchweb',
            'segment_preference': '4',
            'sig': access_token['sig'].encode('utf-8'),
            'token': access_token['token'].encode('utf-8'),
        }
        formats = self._extract_m3u8_formats(
            '%s/api/channel/hls/%s.m3u8?%s'
            % (self._USHER_BASE, channel_id, compat_urllib_parse.urlencode(query)),
            channel_id, 'mp4')
        self._prefer_source(formats)

        view_count = stream.get('viewers')
        timestamp = parse_iso8601(stream.get('created_at'))

        channel = stream['channel']
        title = self._live_title(channel.get('display_name') or channel.get('name'))
        description = channel.get('status')

        thumbnails = []
        for thumbnail_key, thumbnail_url in stream['preview'].items():
            m = re.search(r'(?P<width>\d+)x(?P<height>\d+)\.jpg$', thumbnail_key)
            if not m:
                continue
            thumbnails.append({
                'url': thumbnail_url,
                'width': int(m.group('width')),
                'height': int(m.group('height')),
            })

        return {
            'id': compat_str(stream['_id']),
            'display_id': channel_id,
            'title': title,
            'description': description,
            'thumbnails': thumbnails,
            'uploader': channel.get('display_name'),
            'uploader_id': channel.get('name'),
            'timestamp': timestamp,
            'view_count': view_count,
            'formats': formats,
            'is_live': True,
        }
Commit	Line	Data
04d02a9d	1	# coding: utf-8
ee1e1996 PH	2	from __future__ import unicode_literals
ee1e1996 PH	3
3182f3e2	4	import itertools
79e93125	5	import re
f353cbdb	6	import random
79e93125 PH	7
79e93125 PH	8	from .common import InfoExtractor
1cc79574	9	from ..compat import (
240b9b7a	10	compat_str,
04d02a9d S	11	compat_urllib_parse,
04d02a9d S	12	compat_urllib_request,
1cc79574 PH	13	)
1cc79574 PH	14	from ..utils import (
79e93125	15	ExtractorError,
355d074f	16	parse_iso8601,
79e93125 PH	17	)
	18
	19
c5db6bb3	20	class TwitchBaseIE(InfoExtractor):
240b9b7a	21	_VALID_URL_BASE = r'https?://(?:www\.)?twitch\.tv'
c5db6bb3	22
46fd0dd5	23	_API_BASE = 'https://api.twitch.tv'
240b9b7a	24	_USHER_BASE = 'http://usher.twitch.tv'
04d02a9d	25	_LOGIN_URL = 'https://secure.twitch.tv/user/login'
499bfcbf	26	_NETRC_MACHINE = 'twitch'
79e93125	27
355d074f S	28	def _handle_error(self, response):
	29	if not isinstance(response, dict):
	30	return
	31	error = response.get('error')
	32	if error:
	33	raise ExtractorError(
	34	'%s returned error: %s - %s' % (self.IE_NAME, error, response.get('message')),
	35	expected=True)
	36
	37	def _download_json(self, url, video_id, note='Downloading JSON metadata'):
0d103de3 S	38	headers = {
	39	'Referer': 'http://api.twitch.tv/crossdomain/receiver.html?v=2',
	40	'X-Requested-With': 'XMLHttpRequest',
	41	}
	42	for cookie in self._downloader.cookiejar:
	43	if cookie.name == 'api_token':
	44	headers['Twitch-Api-Token'] = cookie.value
	45	request = compat_urllib_request.Request(url, headers=headers)
	46	response = super(TwitchBaseIE, self)._download_json(request, video_id, note)
355d074f S	47	self._handle_error(response)
	48	return response
	49
c5db6bb3 S	50	def _real_initialize(self):
	51	self._login()
	52
	53	def _login(self):
	54	(username, password) = self._get_login_info()
	55	if username is None:
	56	return
	57
	58	login_page = self._download_webpage(
	59	self._LOGIN_URL, None, 'Downloading login page')
	60
	61	authenticity_token = self._search_regex(
	62	r'<input name="authenticity_token" type="hidden" value="([^"]+)"',
	63	login_page, 'authenticity token')
	64
	65	login_form = {
	66	'utf8': '✓'.encode('utf-8'),
	67	'authenticity_token': authenticity_token,
	68	'redirect_on_login': '',
	69	'embed_form': 'false',
	70	'mp_source_action': '',
	71	'follow': '',
	72	'user[login]': username,
	73	'user[password]': password,
355d074f	74	}
c5db6bb3 S	75
	76	request = compat_urllib_request.Request(
	77	self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
	78	request.add_header('Referer', self._LOGIN_URL)
	79	response = self._download_webpage(
	80	request, None, 'Logging in as %s' % username)
	81
	82	m = re.search(
	83	r"id=([\"'])login_error_message\1[^>]*>(?P<msg>[^<]+)", response)
	84	if m:
	85	raise ExtractorError(
	86	'Unable to login: %s' % m.group('msg').strip(), expected=True)
	87
d0e958c7 PH	88	def _prefer_source(self, formats):
	89	try:
	90	source = next(f for f in formats if f['format_id'] == 'Source')
	91	source['preference'] = 10
	92	except StopIteration:
	93	pass # No Source stream present
	94	self._sort_formats(formats)
	95
c5db6bb3 S	96
	97	class TwitchItemBaseIE(TwitchBaseIE):
	98	def _download_info(self, item, item_id):
	99	return self._extract_info(self._download_json(
355d074f	100	'%s/kraken/videos/%s%s' % (self._API_BASE, item, item_id), item_id,
c5db6bb3	101	'Downloading %s info JSON' % self._ITEM_TYPE))
a22524b0	102
c5db6bb3 S	103	def _extract_media(self, item_id):
c5db6bb3 S	104	info = self._download_info(self._ITEM_SHORTCUT, item_id)
355d074f	105	response = self._download_json(
c5db6bb3 S	106	'%s/api/videos/%s%s' % (self._API_BASE, self._ITEM_SHORTCUT, item_id), item_id,
c5db6bb3 S	107	'Downloading %s playlist JSON' % self._ITEM_TYPE)
355d074f S	108	entries = []
	109	chunks = response['chunks']
	110	qualities = list(chunks.keys())
	111	for num, fragment in enumerate(zip(*chunks.values()), start=1):
	112	formats = []
	113	for fmt_num, fragment_fmt in enumerate(fragment):
	114	format_id = qualities[fmt_num]
	115	fmt = {
	116	'url': fragment_fmt['url'],
	117	'format_id': format_id,
	118	'quality': 1 if format_id == 'live' else 0,
	119	}
	120	m = re.search(r'^(?P<height>\d+)[Pp]', format_id)
	121	if m:
	122	fmt['height'] = int(m.group('height'))
	123	formats.append(fmt)
	124	self._sort_formats(formats)
	125	entry = dict(info)
159444a6	126	entry['id'] = '%s_%d' % (entry['id'], num)
355d074f S	127	entry['title'] = '%s part %d' % (entry['title'], num)
	128	entry['formats'] = formats
	129	entries.append(entry)
013bfdd8	130	return self.playlist_result(entries, info['id'], info['title'])
355d074f S	131
	132	def _extract_info(self, info):
	133	return {
	134	'id': info['_id'],
	135	'title': info['title'],
	136	'description': info['description'],
	137	'duration': info['length'],
	138	'thumbnail': info['preview'],
	139	'uploader': info['channel']['display_name'],
	140	'uploader_id': info['channel']['name'],
	141	'timestamp': parse_iso8601(info['recorded_at']),
	142	'view_count': info['views'],
	143	}
	144
c5db6bb3 S	145	def _real_extract(self, url):
c5db6bb3 S	146	return self._extract_media(self._match_id(url))
04d02a9d	147
04d02a9d	148
c5db6bb3 S	149	class TwitchVideoIE(TwitchItemBaseIE):
	150	IE_NAME = 'twitch:video'
	151	_VALID_URL = r'%s/[^/]+/b/(?P<id>[^/]+)' % TwitchBaseIE._VALID_URL_BASE
	152	_ITEM_TYPE = 'video'
	153	_ITEM_SHORTCUT = 'a'
04d02a9d	154
c5db6bb3 S	155	_TEST = {
	156	'url': 'http://www.twitch.tv/riotgames/b/577357806',
	157	'info_dict': {
	158	'id': 'a577357806',
	159	'title': 'Worlds Semifinals - Star Horn Royal Club vs. OMG',
	160	},
	161	'playlist_mincount': 12,
	162	}
04d02a9d	163
04d02a9d	164
c5db6bb3 S	165	class TwitchChapterIE(TwitchItemBaseIE):
	166	IE_NAME = 'twitch:chapter'
	167	_VALID_URL = r'%s/[^/]+/c/(?P<id>[^/]+)' % TwitchBaseIE._VALID_URL_BASE
	168	_ITEM_TYPE = 'chapter'
	169	_ITEM_SHORTCUT = 'c'
04d02a9d	170
78111136	171	_TESTS = [{
c5db6bb3 S	172	'url': 'http://www.twitch.tv/acracingleague/c/5285812',
	173	'info_dict': {
	174	'id': 'c5285812',
	175	'title': 'ACRL Off Season - Sports Cars @ Nordschleife',
	176	},
	177	'playlist_mincount': 3,
78111136 PH	178	}, {
	179	'url': 'http://www.twitch.tv/tsm_theoddone/c/2349361',
	180	'only_matching': True,
	181	}]
c5db6bb3 S	182
	183
	184	class TwitchVodIE(TwitchItemBaseIE):
	185	IE_NAME = 'twitch:vod'
	186	_VALID_URL = r'%s/[^/]+/v/(?P<id>[^/]+)' % TwitchBaseIE._VALID_URL_BASE
	187	_ITEM_TYPE = 'vod'
	188	_ITEM_SHORTCUT = 'v'
	189
	190	_TEST = {
	191	'url': 'http://www.twitch.tv/ksptv/v/3622000',
	192	'info_dict': {
	193	'id': 'v3622000',
	194	'ext': 'mp4',
	195	'title': '''KSPTV: Squadcast: "Everyone's on vacation so here's Dahud" Edition!''',
	196	'thumbnail': 're:^https?://.*\.jpg$',
	197	'duration': 6951,
	198	'timestamp': 1419028564,
	199	'upload_date': '20141219',
	200	'uploader': 'KSPTV',
	201	'uploader_id': 'ksptv',
	202	'view_count': int,
	203	},
	204	'params': {
	205	# m3u8 download
	206	'skip_download': True,
	207	},
	208	}
04d02a9d	209
79e93125	210	def _real_extract(self, url):
c5db6bb3 S	211	item_id = self._match_id(url)
	212	info = self._download_info(self._ITEM_SHORTCUT, item_id)
	213	access_token = self._download_json(
	214	'%s/api/vods/%s/access_token' % (self._API_BASE, item_id), item_id,
	215	'Downloading %s access token' % self._ITEM_TYPE)
	216	formats = self._extract_m3u8_formats(
240b9b7a S	217	'%s/vod/%s?nauth=%s&nauthsig=%s'
240b9b7a S	218	% (self._USHER_BASE, item_id, access_token['token'], access_token['sig']),
c5db6bb3	219	item_id, 'mp4')
d0e958c7	220	self._prefer_source(formats)
c5db6bb3 S	221	info['formats'] = formats
	222	return info
	223
	224
	225	class TwitchPlaylistBaseIE(TwitchBaseIE):
	226	_PLAYLIST_URL = '%s/kraken/channels/%%s/videos/?offset=%%d&limit=%%d' % TwitchBaseIE._API_BASE
	227	_PAGE_LIMIT = 100
	228
	229	def _extract_playlist(self, channel_id):
	230	info = self._download_json(
	231	'%s/kraken/channels/%s' % (self._API_BASE, channel_id),
	232	channel_id, 'Downloading channel info JSON')
	233	channel_name = info.get('display_name') or info.get('name')
	234	entries = []
	235	offset = 0
	236	limit = self._PAGE_LIMIT
	237	for counter in itertools.count(1):
	238	response = self._download_json(
	239	self._PLAYLIST_URL % (channel_id, offset, limit),
	240	channel_id, 'Downloading %s videos JSON page %d' % (self._PLAYLIST_TYPE, counter))
c2e64f71 S	241	page_entries = self._extract_playlist_page(response)
c2e64f71 S	242	if not page_entries:
c5db6bb3	243	break
c2e64f71	244	entries.extend(page_entries)
c5db6bb3	245	offset += limit
c2e64f71 S	246	return self.playlist_result(
	247	[self.url_result(entry) for entry in set(entries)],
	248	channel_id, channel_name)
	249
	250	def _extract_playlist_page(self, response):
	251	videos = response.get('videos')
	252	return [video['url'] for video in videos] if videos else []
c5db6bb3 S	253
	254	def _real_extract(self, url):
	255	return self._extract_playlist(self._match_id(url))
	256
	257
	258	class TwitchProfileIE(TwitchPlaylistBaseIE):
	259	IE_NAME = 'twitch:profile'
	260	_VALID_URL = r'%s/(?P<id>[^/]+)/profile/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
	261	_PLAYLIST_TYPE = 'profile'
	262
	263	_TEST = {
	264	'url': 'http://www.twitch.tv/vanillatv/profile',
	265	'info_dict': {
	266	'id': 'vanillatv',
	267	'title': 'VanillaTV',
	268	},
	269	'playlist_mincount': 412,
	270	}
	271
	272
	273	class TwitchPastBroadcastsIE(TwitchPlaylistBaseIE):
8bb1bdfa	274	IE_NAME = 'twitch:past_broadcasts'
c5db6bb3 S	275	_VALID_URL = r'%s/(?P<id>[^/]+)/profile/past_broadcasts/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
	276	_PLAYLIST_URL = TwitchPlaylistBaseIE._PLAYLIST_URL + '&broadcasts=true'
	277	_PLAYLIST_TYPE = 'past broadcasts'
	278
	279	_TEST = {
	280	'url': 'http://www.twitch.tv/spamfish/profile/past_broadcasts',
	281	'info_dict': {
	282	'id': 'spamfish',
	283	'title': 'Spamfish',
	284	},
	285	'playlist_mincount': 54,
	286	}
240b9b7a S	287
240b9b7a S	288
c2e64f71 S	289	class TwitchBookmarksIE(TwitchPlaylistBaseIE):
	290	IE_NAME = 'twitch:bookmarks'
	291	_VALID_URL = r'%s/(?P<id>[^/]+)/profile/bookmarks/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
	292	_PLAYLIST_URL = '%s/api/bookmark/?user=%%s&offset=%%d&limit=%%d' % TwitchBaseIE._API_BASE
	293	_PLAYLIST_TYPE = 'bookmarks'
	294
	295	_TEST = {
	296	'url': 'http://www.twitch.tv/ognos/profile/bookmarks',
	297	'info_dict': {
	298	'id': 'ognos',
	299	'title': 'Ognos',
	300	},
	301	'playlist_mincount': 3,
	302	}
	303
	304	def _extract_playlist_page(self, response):
	305	entries = []
	306	for bookmark in response.get('bookmarks', []):
	307	video = bookmark.get('video')
	308	if not video:
	309	continue
	310	entries.append(video['url'])
	311	return entries
	312
	313
240b9b7a S	314	class TwitchStreamIE(TwitchBaseIE):
	315	IE_NAME = 'twitch:stream'
	316	_VALID_URL = r'%s/(?P<id>[^/]+)/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
	317
	318	_TEST = {
	319	'url': 'http://www.twitch.tv/shroomztv',
	320	'info_dict': {
	321	'id': '12772022048',
	322	'display_id': 'shroomztv',
	323	'ext': 'mp4',
	324	'title': 're:^ShroomzTV [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
	325	'description': 'H1Z1 - lonewolfing with ShroomzTV \| A3 Battle Royale later - @ShroomzTV',
	326	'is_live': True,
	327	'timestamp': 1421928037,
	328	'upload_date': '20150122',
	329	'uploader': 'ShroomzTV',
	330	'uploader_id': 'shroomztv',
	331	'view_count': int,
	332	},
	333	'params': {
	334	# m3u8 download
	335	'skip_download': True,
	336	},
	337	}
	338
	339	def _real_extract(self, url):
	340	channel_id = self._match_id(url)
	341
	342	stream = self._download_json(
	343	'%s/kraken/streams/%s' % (self._API_BASE, channel_id), channel_id,
	344	'Downloading stream JSON').get('stream')
	345
	346	# Fallback on profile extraction if stream is offline
	347	if not stream:
	348	return self.url_result(
	349	'http://www.twitch.tv/%s/profile' % channel_id,
	350	'TwitchProfile', channel_id)
	351
	352	access_token = self._download_json(
	353	'%s/api/channels/%s/access_token' % (self._API_BASE, channel_id), channel_id,
	354	'Downloading channel access token')
	355
	356	query = {
	357	'allow_source': 'true',
f353cbdb	358	'p': random.randint(1000000, 10000000),
240b9b7a S	359	'player': 'twitchweb',
240b9b7a S	360	'segment_preference': '4',
d34e7949 S	361	'sig': access_token['sig'].encode('utf-8'),
d34e7949 S	362	'token': access_token['token'].encode('utf-8'),
240b9b7a	363	}
240b9b7a S	364	formats = self._extract_m3u8_formats(
240b9b7a S	365	'%s/api/channel/hls/%s.m3u8?%s'
d34e7949	366	% (self._USHER_BASE, channel_id, compat_urllib_parse.urlencode(query)),
240b9b7a	367	channel_id, 'mp4')
d0e958c7	368	self._prefer_source(formats)
240b9b7a S	369
	370	view_count = stream.get('viewers')
	371	timestamp = parse_iso8601(stream.get('created_at'))
	372
	373	channel = stream['channel']
	374	title = self._live_title(channel.get('display_name') or channel.get('name'))
	375	description = channel.get('status')
	376
	377	thumbnails = []
	378	for thumbnail_key, thumbnail_url in stream['preview'].items():
	379	m = re.search(r'(?P<width>\d+)x(?P<height>\d+)\.jpg$', thumbnail_key)
	380	if not m:
	381	continue
	382	thumbnails.append({
	383	'url': thumbnail_url,
	384	'width': int(m.group('width')),
	385	'height': int(m.group('height')),
	386	})
	387
	388	return {
	389	'id': compat_str(stream['_id']),
	390	'display_id': channel_id,
	391	'title': title,
	392	'description': description,
	393	'thumbnails': thumbnails,
	394	'uploader': channel.get('display_name'),
	395	'uploader_id': channel.get('name'),
	396	'timestamp': timestamp,
	397	'view_count': view_count,
	398	'formats': formats,
	399	'is_live': True,
12d1fb5a	400	}