[yt-dlp.git] / youtube_dl / extractor / twitch.py

# coding: utf-8
from __future__ import unicode_literals

import itertools
import re
import random

from .common import InfoExtractor
from ..compat import (
    compat_str,
    compat_urllib_parse,
    compat_urllib_request,
)
from ..utils import (
    ExtractorError,
    parse_iso8601,
)


class TwitchBaseIE(InfoExtractor):
    _VALID_URL_BASE = r'https?://(?:www\.)?twitch\.tv'

    _API_BASE = 'https://api.twitch.tv'
    _USHER_BASE = 'http://usher.twitch.tv'
    _LOGIN_URL = 'https://secure.twitch.tv/user/login'
    _LOGIN_POST_URL = 'https://secure-login.twitch.tv/login'
    _NETRC_MACHINE = 'twitch'

    def _handle_error(self, response):
        if not isinstance(response, dict):
            return
        error = response.get('error')
        if error:
            raise ExtractorError(
                '%s returned error: %s - %s' % (self.IE_NAME, error, response.get('message')),
                expected=True)

    def _download_json(self, url, video_id, note='Downloading JSON metadata'):
        headers = {
            'Referer': 'http://api.twitch.tv/crossdomain/receiver.html?v=2',
            'X-Requested-With': 'XMLHttpRequest',
        }
        for cookie in self._downloader.cookiejar:
            if cookie.name == 'api_token':
                headers['Twitch-Api-Token'] = cookie.value
        request = compat_urllib_request.Request(url, headers=headers)
        response = super(TwitchBaseIE, self)._download_json(request, video_id, note)
        self._handle_error(response)
        return response

    def _real_initialize(self):
        self._login()

    def _login(self):
        (username, password) = self._get_login_info()
        if username is None:
            return

        login_page = self._download_webpage(
            self._LOGIN_URL, None, 'Downloading login page')

        authenticity_token = self._search_regex(
            r'<input name="authenticity_token" type="hidden" value="([^"]+)"',
            login_page, 'authenticity token')

        login_form = {
            'utf8': '✓'.encode('utf-8'),
            'authenticity_token': authenticity_token,
            'redirect_on_login': '',
            'embed_form': 'false',
            'mp_source_action': 'login-button',
            'follow': '',
            'login': username,
            'password': password,
        }

        request = compat_urllib_request.Request(
            self._LOGIN_POST_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
        request.add_header('Referer', self._LOGIN_URL)
        response = self._download_webpage(
            request, None, 'Logging in as %s' % username)

        m = re.search(
            r"id=([\"'])login_error_message\1[^>]*>(?P<msg>[^<]+)", response)
        if m:
            raise ExtractorError(
                'Unable to login: %s' % m.group('msg').strip(), expected=True)

    def _prefer_source(self, formats):
        try:
            source = next(f for f in formats if f['format_id'] == 'Source')
            source['preference'] = 10
        except StopIteration:
            pass  # No Source stream present
        self._sort_formats(formats)


class TwitchItemBaseIE(TwitchBaseIE):
    def _download_info(self, item, item_id):
        return self._extract_info(self._download_json(
            '%s/kraken/videos/%s%s' % (self._API_BASE, item, item_id), item_id,
            'Downloading %s info JSON' % self._ITEM_TYPE))

    def _extract_media(self, item_id):
        info = self._download_info(self._ITEM_SHORTCUT, item_id)
        response = self._download_json(
            '%s/api/videos/%s%s' % (self._API_BASE, self._ITEM_SHORTCUT, item_id), item_id,
            'Downloading %s playlist JSON' % self._ITEM_TYPE)
        entries = []
        chunks = response['chunks']
        qualities = list(chunks.keys())
        for num, fragment in enumerate(zip(*chunks.values()), start=1):
            formats = []
            for fmt_num, fragment_fmt in enumerate(fragment):
                format_id = qualities[fmt_num]
                fmt = {
                    'url': fragment_fmt['url'],
                    'format_id': format_id,
                    'quality': 1 if format_id == 'live' else 0,
                }
                m = re.search(r'^(?P<height>\d+)[Pp]', format_id)
                if m:
                    fmt['height'] = int(m.group('height'))
                formats.append(fmt)
            self._sort_formats(formats)
            entry = dict(info)
            entry['id'] = '%s_%d' % (entry['id'], num)
            entry['title'] = '%s part %d' % (entry['title'], num)
            entry['formats'] = formats
            entries.append(entry)
        return self.playlist_result(entries, info['id'], info['title'])

    def _extract_info(self, info):
        return {
            'id': info['_id'],
            'title': info['title'],
            'description': info['description'],
            'duration': info['length'],
            'thumbnail': info['preview'],
            'uploader': info['channel']['display_name'],
            'uploader_id': info['channel']['name'],
            'timestamp': parse_iso8601(info['recorded_at']),
            'view_count': info['views'],
        }

    def _real_extract(self, url):
        return self._extract_media(self._match_id(url))


class TwitchVideoIE(TwitchItemBaseIE):
    IE_NAME = 'twitch:video'
    _VALID_URL = r'%s/[^/]+/b/(?P<id>[^/]+)' % TwitchBaseIE._VALID_URL_BASE
    _ITEM_TYPE = 'video'
    _ITEM_SHORTCUT = 'a'

    _TEST = {
        'url': 'http://www.twitch.tv/riotgames/b/577357806',
        'info_dict': {
            'id': 'a577357806',
            'title': 'Worlds Semifinals - Star Horn Royal Club vs. OMG',
        },
        'playlist_mincount': 12,
    }


class TwitchChapterIE(TwitchItemBaseIE):
    IE_NAME = 'twitch:chapter'
    _VALID_URL = r'%s/[^/]+/c/(?P<id>[^/]+)' % TwitchBaseIE._VALID_URL_BASE
    _ITEM_TYPE = 'chapter'
    _ITEM_SHORTCUT = 'c'

    _TESTS = [{
        'url': 'http://www.twitch.tv/acracingleague/c/5285812',
        'info_dict': {
            'id': 'c5285812',
            'title': 'ACRL Off Season - Sports Cars @ Nordschleife',
        },
        'playlist_mincount': 3,
    }, {
        'url': 'http://www.twitch.tv/tsm_theoddone/c/2349361',
        'only_matching': True,
    }]


class TwitchVodIE(TwitchItemBaseIE):
    IE_NAME = 'twitch:vod'
    _VALID_URL = r'%s/[^/]+/v/(?P<id>[^/]+)' % TwitchBaseIE._VALID_URL_BASE
    _ITEM_TYPE = 'vod'
    _ITEM_SHORTCUT = 'v'

    _TEST = {
        'url': 'http://www.twitch.tv/ksptv/v/3622000',
        'info_dict': {
            'id': 'v3622000',
            'ext': 'mp4',
            'title': '''KSPTV: Squadcast: "Everyone's on vacation so here's Dahud" Edition!''',
            'thumbnail': 're:^https?://.*\.jpg$',
            'duration': 6951,
            'timestamp': 1419028564,
            'upload_date': '20141219',
            'uploader': 'KSPTV',
            'uploader_id': 'ksptv',
            'view_count': int,
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }

    def _real_extract(self, url):
        item_id = self._match_id(url)
        info = self._download_info(self._ITEM_SHORTCUT, item_id)
        access_token = self._download_json(
            '%s/api/vods/%s/access_token' % (self._API_BASE, item_id), item_id,
            'Downloading %s access token' % self._ITEM_TYPE)
        formats = self._extract_m3u8_formats(
            '%s/vod/%s?nauth=%s&nauthsig=%s'
            % (self._USHER_BASE, item_id, access_token['token'], access_token['sig']),
            item_id, 'mp4')
        self._prefer_source(formats)
        info['formats'] = formats
        return info


class TwitchPlaylistBaseIE(TwitchBaseIE):
    _PLAYLIST_URL = '%s/kraken/channels/%%s/videos/?offset=%%d&limit=%%d' % TwitchBaseIE._API_BASE
    _PAGE_LIMIT = 100

    def _extract_playlist(self, channel_id):
        info = self._download_json(
            '%s/kraken/channels/%s' % (self._API_BASE, channel_id),
            channel_id, 'Downloading channel info JSON')
        channel_name = info.get('display_name') or info.get('name')
        entries = []
        offset = 0
        limit = self._PAGE_LIMIT
        for counter in itertools.count(1):
            response = self._download_json(
                self._PLAYLIST_URL % (channel_id, offset, limit),
                channel_id, 'Downloading %s videos JSON page %d' % (self._PLAYLIST_TYPE, counter))
            page_entries = self._extract_playlist_page(response)
            if not page_entries:
                break
            entries.extend(page_entries)
            offset += limit
        return self.playlist_result(
            [self.url_result(entry) for entry in set(entries)],
            channel_id, channel_name)

    def _extract_playlist_page(self, response):
        videos = response.get('videos')
        return [video['url'] for video in videos] if videos else []

    def _real_extract(self, url):
        return self._extract_playlist(self._match_id(url))


class TwitchProfileIE(TwitchPlaylistBaseIE):
    IE_NAME = 'twitch:profile'
    _VALID_URL = r'%s/(?P<id>[^/]+)/profile/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
    _PLAYLIST_TYPE = 'profile'

    _TEST = {
        'url': 'http://www.twitch.tv/vanillatv/profile',
        'info_dict': {
            'id': 'vanillatv',
            'title': 'VanillaTV',
        },
        'playlist_mincount': 412,
    }


class TwitchPastBroadcastsIE(TwitchPlaylistBaseIE):
    IE_NAME = 'twitch:past_broadcasts'
    _VALID_URL = r'%s/(?P<id>[^/]+)/profile/past_broadcasts/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
    _PLAYLIST_URL = TwitchPlaylistBaseIE._PLAYLIST_URL + '&broadcasts=true'
    _PLAYLIST_TYPE = 'past broadcasts'

    _TEST = {
        'url': 'http://www.twitch.tv/spamfish/profile/past_broadcasts',
        'info_dict': {
            'id': 'spamfish',
            'title': 'Spamfish',
        },
        'playlist_mincount': 54,
    }


class TwitchBookmarksIE(TwitchPlaylistBaseIE):
    IE_NAME = 'twitch:bookmarks'
    _VALID_URL = r'%s/(?P<id>[^/]+)/profile/bookmarks/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
    _PLAYLIST_URL = '%s/api/bookmark/?user=%%s&offset=%%d&limit=%%d' % TwitchBaseIE._API_BASE
    _PLAYLIST_TYPE = 'bookmarks'

    _TEST = {
        'url': 'http://www.twitch.tv/ognos/profile/bookmarks',
        'info_dict': {
            'id': 'ognos',
            'title': 'Ognos',
        },
        'playlist_mincount': 3,
    }

    def _extract_playlist_page(self, response):
        entries = []
        for bookmark in response.get('bookmarks', []):
            video = bookmark.get('video')
            if not video:
                continue
            entries.append(video['url'])
        return entries


class TwitchStreamIE(TwitchBaseIE):
    IE_NAME = 'twitch:stream'
    _VALID_URL = r'%s/(?P<id>[^/]+)/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE

    _TEST = {
        'url': 'http://www.twitch.tv/shroomztv',
        'info_dict': {
            'id': '12772022048',
            'display_id': 'shroomztv',
            'ext': 'mp4',
            'title': 're:^ShroomzTV [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
            'description': 'H1Z1 - lonewolfing with ShroomzTV | A3 Battle Royale later - @ShroomzTV',
            'is_live': True,
            'timestamp': 1421928037,
            'upload_date': '20150122',
            'uploader': 'ShroomzTV',
            'uploader_id': 'shroomztv',
            'view_count': int,
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }

    def _real_extract(self, url):
        channel_id = self._match_id(url)

        stream = self._download_json(
            '%s/kraken/streams/%s' % (self._API_BASE, channel_id), channel_id,
            'Downloading stream JSON').get('stream')

        # Fallback on profile extraction if stream is offline
        if not stream:
            return self.url_result(
                'http://www.twitch.tv/%s/profile' % channel_id,
                'TwitchProfile', channel_id)

        access_token = self._download_json(
            '%s/api/channels/%s/access_token' % (self._API_BASE, channel_id), channel_id,
            'Downloading channel access token')

        query = {
            'allow_source': 'true',
            'p': random.randint(1000000, 10000000),
            'player': 'twitchweb',
            'segment_preference': '4',
            'sig': access_token['sig'].encode('utf-8'),
            'token': access_token['token'].encode('utf-8'),
        }
        formats = self._extract_m3u8_formats(
            '%s/api/channel/hls/%s.m3u8?%s'
            % (self._USHER_BASE, channel_id, compat_urllib_parse.urlencode(query)),
            channel_id, 'mp4')
        self._prefer_source(formats)

        view_count = stream.get('viewers')
        timestamp = parse_iso8601(stream.get('created_at'))

        channel = stream['channel']
        title = self._live_title(channel.get('display_name') or channel.get('name'))
        description = channel.get('status')

        thumbnails = []
        for thumbnail_key, thumbnail_url in stream['preview'].items():
            m = re.search(r'(?P<width>\d+)x(?P<height>\d+)\.jpg$', thumbnail_key)
            if not m:
                continue
            thumbnails.append({
                'url': thumbnail_url,
                'width': int(m.group('width')),
                'height': int(m.group('height')),
            })

        return {
            'id': compat_str(stream['_id']),
            'display_id': channel_id,
            'title': title,
            'description': description,
            'thumbnails': thumbnails,
            'uploader': channel.get('display_name'),
            'uploader_id': channel.get('name'),
            'timestamp': timestamp,
            'view_count': view_count,
            'formats': formats,
            'is_live': True,
        }
Commit	Line	Data
04d02a9d	1	# coding: utf-8
ee1e1996 PH	2	from __future__ import unicode_literals
ee1e1996 PH	3
3182f3e2	4	import itertools
79e93125	5	import re
f353cbdb	6	import random
79e93125 PH	7
79e93125 PH	8	from .common import InfoExtractor
1cc79574	9	from ..compat import (
240b9b7a	10	compat_str,
04d02a9d S	11	compat_urllib_parse,
04d02a9d S	12	compat_urllib_request,
1cc79574 PH	13	)
1cc79574 PH	14	from ..utils import (
79e93125	15	ExtractorError,
355d074f	16	parse_iso8601,
79e93125 PH	17	)
	18
	19
c5db6bb3	20	class TwitchBaseIE(InfoExtractor):
240b9b7a	21	_VALID_URL_BASE = r'https?://(?:www\.)?twitch\.tv'
c5db6bb3	22
46fd0dd5	23	_API_BASE = 'https://api.twitch.tv'
240b9b7a	24	_USHER_BASE = 'http://usher.twitch.tv'
04d02a9d	25	_LOGIN_URL = 'https://secure.twitch.tv/user/login'
1887ecd4	26	_LOGIN_POST_URL = 'https://secure-login.twitch.tv/login'
499bfcbf	27	_NETRC_MACHINE = 'twitch'
79e93125	28
355d074f S	29	def _handle_error(self, response):
	30	if not isinstance(response, dict):
	31	return
	32	error = response.get('error')
	33	if error:
	34	raise ExtractorError(
	35	'%s returned error: %s - %s' % (self.IE_NAME, error, response.get('message')),
	36	expected=True)
	37
	38	def _download_json(self, url, video_id, note='Downloading JSON metadata'):
0d103de3 S	39	headers = {
	40	'Referer': 'http://api.twitch.tv/crossdomain/receiver.html?v=2',
	41	'X-Requested-With': 'XMLHttpRequest',
	42	}
	43	for cookie in self._downloader.cookiejar:
	44	if cookie.name == 'api_token':
	45	headers['Twitch-Api-Token'] = cookie.value
	46	request = compat_urllib_request.Request(url, headers=headers)
	47	response = super(TwitchBaseIE, self)._download_json(request, video_id, note)
355d074f S	48	self._handle_error(response)
	49	return response
	50
c5db6bb3 S	51	def _real_initialize(self):
	52	self._login()
	53
	54	def _login(self):
	55	(username, password) = self._get_login_info()
	56	if username is None:
	57	return
	58
	59	login_page = self._download_webpage(
	60	self._LOGIN_URL, None, 'Downloading login page')
	61
	62	authenticity_token = self._search_regex(
	63	r'<input name="authenticity_token" type="hidden" value="([^"]+)"',
	64	login_page, 'authenticity token')
	65
	66	login_form = {
	67	'utf8': '✓'.encode('utf-8'),
	68	'authenticity_token': authenticity_token,
	69	'redirect_on_login': '',
	70	'embed_form': 'false',
1887ecd4	71	'mp_source_action': 'login-button',
c5db6bb3	72	'follow': '',
1887ecd4 S	73	'login': username,
1887ecd4 S	74	'password': password,
355d074f	75	}
c5db6bb3 S	76
c5db6bb3 S	77	request = compat_urllib_request.Request(
1887ecd4	78	self._LOGIN_POST_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
c5db6bb3 S	79	request.add_header('Referer', self._LOGIN_URL)
	80	response = self._download_webpage(
	81	request, None, 'Logging in as %s' % username)
	82
	83	m = re.search(
	84	r"id=([\"'])login_error_message\1[^>]*>(?P<msg>[^<]+)", response)
	85	if m:
	86	raise ExtractorError(
	87	'Unable to login: %s' % m.group('msg').strip(), expected=True)
	88
d0e958c7 PH	89	def _prefer_source(self, formats):
	90	try:
	91	source = next(f for f in formats if f['format_id'] == 'Source')
	92	source['preference'] = 10
	93	except StopIteration:
	94	pass # No Source stream present
	95	self._sort_formats(formats)
	96
c5db6bb3 S	97
	98	class TwitchItemBaseIE(TwitchBaseIE):
	99	def _download_info(self, item, item_id):
	100	return self._extract_info(self._download_json(
355d074f	101	'%s/kraken/videos/%s%s' % (self._API_BASE, item, item_id), item_id,
c5db6bb3	102	'Downloading %s info JSON' % self._ITEM_TYPE))
a22524b0	103
c5db6bb3 S	104	def _extract_media(self, item_id):
c5db6bb3 S	105	info = self._download_info(self._ITEM_SHORTCUT, item_id)
355d074f	106	response = self._download_json(
c5db6bb3 S	107	'%s/api/videos/%s%s' % (self._API_BASE, self._ITEM_SHORTCUT, item_id), item_id,
c5db6bb3 S	108	'Downloading %s playlist JSON' % self._ITEM_TYPE)
355d074f S	109	entries = []
	110	chunks = response['chunks']
	111	qualities = list(chunks.keys())
	112	for num, fragment in enumerate(zip(*chunks.values()), start=1):
	113	formats = []
	114	for fmt_num, fragment_fmt in enumerate(fragment):
	115	format_id = qualities[fmt_num]
	116	fmt = {
	117	'url': fragment_fmt['url'],
	118	'format_id': format_id,
	119	'quality': 1 if format_id == 'live' else 0,
	120	}
	121	m = re.search(r'^(?P<height>\d+)[Pp]', format_id)
	122	if m:
	123	fmt['height'] = int(m.group('height'))
	124	formats.append(fmt)
	125	self._sort_formats(formats)
	126	entry = dict(info)
159444a6	127	entry['id'] = '%s_%d' % (entry['id'], num)
355d074f S	128	entry['title'] = '%s part %d' % (entry['title'], num)
	129	entry['formats'] = formats
	130	entries.append(entry)
013bfdd8	131	return self.playlist_result(entries, info['id'], info['title'])
355d074f S	132
	133	def _extract_info(self, info):
	134	return {
	135	'id': info['_id'],
	136	'title': info['title'],
	137	'description': info['description'],
	138	'duration': info['length'],
	139	'thumbnail': info['preview'],
	140	'uploader': info['channel']['display_name'],
	141	'uploader_id': info['channel']['name'],
	142	'timestamp': parse_iso8601(info['recorded_at']),
	143	'view_count': info['views'],
	144	}
	145
c5db6bb3 S	146	def _real_extract(self, url):
c5db6bb3 S	147	return self._extract_media(self._match_id(url))
04d02a9d	148
04d02a9d	149
c5db6bb3 S	150	class TwitchVideoIE(TwitchItemBaseIE):
	151	IE_NAME = 'twitch:video'
	152	_VALID_URL = r'%s/[^/]+/b/(?P<id>[^/]+)' % TwitchBaseIE._VALID_URL_BASE
	153	_ITEM_TYPE = 'video'
	154	_ITEM_SHORTCUT = 'a'
04d02a9d	155
c5db6bb3 S	156	_TEST = {
	157	'url': 'http://www.twitch.tv/riotgames/b/577357806',
	158	'info_dict': {
	159	'id': 'a577357806',
	160	'title': 'Worlds Semifinals - Star Horn Royal Club vs. OMG',
	161	},
	162	'playlist_mincount': 12,
	163	}
04d02a9d	164
04d02a9d	165
c5db6bb3 S	166	class TwitchChapterIE(TwitchItemBaseIE):
	167	IE_NAME = 'twitch:chapter'
	168	_VALID_URL = r'%s/[^/]+/c/(?P<id>[^/]+)' % TwitchBaseIE._VALID_URL_BASE
	169	_ITEM_TYPE = 'chapter'
	170	_ITEM_SHORTCUT = 'c'
04d02a9d	171
78111136	172	_TESTS = [{
c5db6bb3 S	173	'url': 'http://www.twitch.tv/acracingleague/c/5285812',
	174	'info_dict': {
	175	'id': 'c5285812',
	176	'title': 'ACRL Off Season - Sports Cars @ Nordschleife',
	177	},
	178	'playlist_mincount': 3,
78111136 PH	179	}, {
	180	'url': 'http://www.twitch.tv/tsm_theoddone/c/2349361',
	181	'only_matching': True,
	182	}]
c5db6bb3 S	183
	184
	185	class TwitchVodIE(TwitchItemBaseIE):
	186	IE_NAME = 'twitch:vod'
	187	_VALID_URL = r'%s/[^/]+/v/(?P<id>[^/]+)' % TwitchBaseIE._VALID_URL_BASE
	188	_ITEM_TYPE = 'vod'
	189	_ITEM_SHORTCUT = 'v'
	190
	191	_TEST = {
	192	'url': 'http://www.twitch.tv/ksptv/v/3622000',
	193	'info_dict': {
	194	'id': 'v3622000',
	195	'ext': 'mp4',
	196	'title': '''KSPTV: Squadcast: "Everyone's on vacation so here's Dahud" Edition!''',
	197	'thumbnail': 're:^https?://.*\.jpg$',
	198	'duration': 6951,
	199	'timestamp': 1419028564,
	200	'upload_date': '20141219',
	201	'uploader': 'KSPTV',
	202	'uploader_id': 'ksptv',
	203	'view_count': int,
	204	},
	205	'params': {
	206	# m3u8 download
	207	'skip_download': True,
	208	},
	209	}
04d02a9d	210
79e93125	211	def _real_extract(self, url):
c5db6bb3 S	212	item_id = self._match_id(url)
	213	info = self._download_info(self._ITEM_SHORTCUT, item_id)
	214	access_token = self._download_json(
	215	'%s/api/vods/%s/access_token' % (self._API_BASE, item_id), item_id,
	216	'Downloading %s access token' % self._ITEM_TYPE)
	217	formats = self._extract_m3u8_formats(
240b9b7a S	218	'%s/vod/%s?nauth=%s&nauthsig=%s'
240b9b7a S	219	% (self._USHER_BASE, item_id, access_token['token'], access_token['sig']),
c5db6bb3	220	item_id, 'mp4')
d0e958c7	221	self._prefer_source(formats)
c5db6bb3 S	222	info['formats'] = formats
	223	return info
	224
	225
	226	class TwitchPlaylistBaseIE(TwitchBaseIE):
	227	_PLAYLIST_URL = '%s/kraken/channels/%%s/videos/?offset=%%d&limit=%%d' % TwitchBaseIE._API_BASE
	228	_PAGE_LIMIT = 100
	229
	230	def _extract_playlist(self, channel_id):
	231	info = self._download_json(
	232	'%s/kraken/channels/%s' % (self._API_BASE, channel_id),
	233	channel_id, 'Downloading channel info JSON')
	234	channel_name = info.get('display_name') or info.get('name')
	235	entries = []
	236	offset = 0
	237	limit = self._PAGE_LIMIT
	238	for counter in itertools.count(1):
	239	response = self._download_json(
	240	self._PLAYLIST_URL % (channel_id, offset, limit),
	241	channel_id, 'Downloading %s videos JSON page %d' % (self._PLAYLIST_TYPE, counter))
c2e64f71 S	242	page_entries = self._extract_playlist_page(response)
c2e64f71 S	243	if not page_entries:
c5db6bb3	244	break
c2e64f71	245	entries.extend(page_entries)
c5db6bb3	246	offset += limit
c2e64f71 S	247	return self.playlist_result(
	248	[self.url_result(entry) for entry in set(entries)],
	249	channel_id, channel_name)
	250
	251	def _extract_playlist_page(self, response):
	252	videos = response.get('videos')
	253	return [video['url'] for video in videos] if videos else []
c5db6bb3 S	254
	255	def _real_extract(self, url):
	256	return self._extract_playlist(self._match_id(url))
	257
	258
	259	class TwitchProfileIE(TwitchPlaylistBaseIE):
	260	IE_NAME = 'twitch:profile'
	261	_VALID_URL = r'%s/(?P<id>[^/]+)/profile/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
	262	_PLAYLIST_TYPE = 'profile'
	263
	264	_TEST = {
	265	'url': 'http://www.twitch.tv/vanillatv/profile',
	266	'info_dict': {
	267	'id': 'vanillatv',
	268	'title': 'VanillaTV',
	269	},
	270	'playlist_mincount': 412,
	271	}
	272
	273
	274	class TwitchPastBroadcastsIE(TwitchPlaylistBaseIE):
8bb1bdfa	275	IE_NAME = 'twitch:past_broadcasts'
c5db6bb3 S	276	_VALID_URL = r'%s/(?P<id>[^/]+)/profile/past_broadcasts/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
	277	_PLAYLIST_URL = TwitchPlaylistBaseIE._PLAYLIST_URL + '&broadcasts=true'
	278	_PLAYLIST_TYPE = 'past broadcasts'
	279
	280	_TEST = {
	281	'url': 'http://www.twitch.tv/spamfish/profile/past_broadcasts',
	282	'info_dict': {
	283	'id': 'spamfish',
	284	'title': 'Spamfish',
	285	},
	286	'playlist_mincount': 54,
	287	}
240b9b7a S	288
240b9b7a S	289
c2e64f71 S	290	class TwitchBookmarksIE(TwitchPlaylistBaseIE):
	291	IE_NAME = 'twitch:bookmarks'
	292	_VALID_URL = r'%s/(?P<id>[^/]+)/profile/bookmarks/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
	293	_PLAYLIST_URL = '%s/api/bookmark/?user=%%s&offset=%%d&limit=%%d' % TwitchBaseIE._API_BASE
	294	_PLAYLIST_TYPE = 'bookmarks'
	295
	296	_TEST = {
	297	'url': 'http://www.twitch.tv/ognos/profile/bookmarks',
	298	'info_dict': {
	299	'id': 'ognos',
	300	'title': 'Ognos',
	301	},
	302	'playlist_mincount': 3,
	303	}
	304
	305	def _extract_playlist_page(self, response):
	306	entries = []
	307	for bookmark in response.get('bookmarks', []):
	308	video = bookmark.get('video')
	309	if not video:
	310	continue
	311	entries.append(video['url'])
	312	return entries
	313
	314
240b9b7a S	315	class TwitchStreamIE(TwitchBaseIE):
	316	IE_NAME = 'twitch:stream'
	317	_VALID_URL = r'%s/(?P<id>[^/]+)/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
	318
	319	_TEST = {
	320	'url': 'http://www.twitch.tv/shroomztv',
	321	'info_dict': {
	322	'id': '12772022048',
	323	'display_id': 'shroomztv',
	324	'ext': 'mp4',
	325	'title': 're:^ShroomzTV [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
	326	'description': 'H1Z1 - lonewolfing with ShroomzTV \| A3 Battle Royale later - @ShroomzTV',
	327	'is_live': True,
	328	'timestamp': 1421928037,
	329	'upload_date': '20150122',
	330	'uploader': 'ShroomzTV',
	331	'uploader_id': 'shroomztv',
	332	'view_count': int,
	333	},
	334	'params': {
	335	# m3u8 download
	336	'skip_download': True,
	337	},
	338	}
	339
	340	def _real_extract(self, url):
	341	channel_id = self._match_id(url)
	342
	343	stream = self._download_json(
	344	'%s/kraken/streams/%s' % (self._API_BASE, channel_id), channel_id,
	345	'Downloading stream JSON').get('stream')
	346
	347	# Fallback on profile extraction if stream is offline
	348	if not stream:
	349	return self.url_result(
	350	'http://www.twitch.tv/%s/profile' % channel_id,
	351	'TwitchProfile', channel_id)
	352
	353	access_token = self._download_json(
	354	'%s/api/channels/%s/access_token' % (self._API_BASE, channel_id), channel_id,
	355	'Downloading channel access token')
	356
	357	query = {
	358	'allow_source': 'true',
f353cbdb	359	'p': random.randint(1000000, 10000000),
240b9b7a S	360	'player': 'twitchweb',
240b9b7a S	361	'segment_preference': '4',
d34e7949 S	362	'sig': access_token['sig'].encode('utf-8'),
d34e7949 S	363	'token': access_token['token'].encode('utf-8'),
240b9b7a	364	}
240b9b7a S	365	formats = self._extract_m3u8_formats(
240b9b7a S	366	'%s/api/channel/hls/%s.m3u8?%s'
d34e7949	367	% (self._USHER_BASE, channel_id, compat_urllib_parse.urlencode(query)),
240b9b7a	368	channel_id, 'mp4')
d0e958c7	369	self._prefer_source(formats)
240b9b7a S	370
	371	view_count = stream.get('viewers')
	372	timestamp = parse_iso8601(stream.get('created_at'))
	373
	374	channel = stream['channel']
	375	title = self._live_title(channel.get('display_name') or channel.get('name'))
	376	description = channel.get('status')
	377
	378	thumbnails = []
	379	for thumbnail_key, thumbnail_url in stream['preview'].items():
	380	m = re.search(r'(?P<width>\d+)x(?P<height>\d+)\.jpg$', thumbnail_key)
	381	if not m:
	382	continue
	383	thumbnails.append({
	384	'url': thumbnail_url,
	385	'width': int(m.group('width')),
	386	'height': int(m.group('height')),
	387	})
	388
	389	return {
	390	'id': compat_str(stream['_id']),
	391	'display_id': channel_id,
	392	'title': title,
	393	'description': description,
	394	'thumbnails': thumbnails,
	395	'uploader': channel.get('display_name'),
	396	'uploader_id': channel.get('name'),
	397	'timestamp': timestamp,
	398	'view_count': view_count,
	399	'formats': formats,
	400	'is_live': True,
12d1fb5a	401	}