]>
Commit | Line | Data |
---|---|---|
79e93125 PH |
1 | import json |
2 | import os | |
3 | import re | |
79e93125 PH |
4 | |
5 | from .common import InfoExtractor | |
6 | from ..utils import ( | |
7 | ExtractorError, | |
8 | formatSeconds, | |
9 | ) | |
10 | ||
11 | ||
12 | class JustinTVIE(InfoExtractor): | |
13 | """Information extractor for justin.tv and twitch.tv""" | |
14 | # TODO: One broadcast may be split into multiple videos. The key | |
15 | # 'broadcast_id' is the same for all parts, and 'broadcast_part' | |
16 | # starts at 1 and increases. Can we treat all parts as one video? | |
17 | ||
18 | _VALID_URL = r"""(?x)^(?:http://)?(?:www\.)?(?:twitch|justin)\.tv/ | |
19 | (?: | |
20 | (?P<channelid>[^/]+)| | |
21 | (?:(?:[^/]+)/b/(?P<videoid>[^/]+))| | |
22 | (?:(?:[^/]+)/c/(?P<chapterid>[^/]+)) | |
23 | ) | |
24 | /?(?:\#.*)?$ | |
25 | """ | |
26 | _JUSTIN_PAGE_LIMIT = 100 | |
27 | IE_NAME = u'justin.tv' | |
6f5ac90c PH |
28 | _TEST = { |
29 | u'url': u'http://www.twitch.tv/thegamedevhub/b/296128360', | |
30 | u'file': u'296128360.flv', | |
31 | u'md5': u'ecaa8a790c22a40770901460af191c9a', | |
32 | u'info_dict': { | |
33 | u"upload_date": u"20110927", | |
34 | u"uploader_id": 25114803, | |
35 | u"uploader": u"thegamedevhub", | |
36 | u"title": u"Beginner Series - Scripting With Python Pt.1" | |
37 | } | |
38 | } | |
79e93125 PH |
39 | |
40 | def report_download_page(self, channel, offset): | |
41 | """Report attempt to download a single page of videos.""" | |
42 | self.to_screen(u'%s: Downloading video information from %d to %d' % | |
43 | (channel, offset, offset + self._JUSTIN_PAGE_LIMIT)) | |
44 | ||
45 | # Return count of items, list of *valid* items | |
46 | def _parse_page(self, url, video_id): | |
190717e3 PH |
47 | info_json = self._download_webpage(url, video_id, |
48 | u'Downloading video info JSON', | |
49 | u'unable to download video info JSON') | |
79e93125 | 50 | |
190717e3 | 51 | response = json.loads(info_json) |
79e93125 PH |
52 | if type(response) != list: |
53 | error_text = response.get('error', 'unknown error') | |
54 | raise ExtractorError(u'Justin.tv API: %s' % error_text) | |
55 | info = [] | |
56 | for clip in response: | |
57 | video_url = clip['video_file_url'] | |
58 | if video_url: | |
59 | video_extension = os.path.splitext(video_url)[1][1:] | |
60 | video_date = re.sub('-', '', clip['start_time'][:10]) | |
61 | video_uploader_id = clip.get('user_id', clip.get('channel_id')) | |
62 | video_id = clip['id'] | |
63 | video_title = clip.get('title', video_id) | |
64 | info.append({ | |
65 | 'id': video_id, | |
66 | 'url': video_url, | |
67 | 'title': video_title, | |
68 | 'uploader': clip.get('channel_name', video_uploader_id), | |
69 | 'uploader_id': video_uploader_id, | |
70 | 'upload_date': video_date, | |
71 | 'ext': video_extension, | |
72 | }) | |
73 | return (len(response), info) | |
74 | ||
75 | def _real_extract(self, url): | |
76 | mobj = re.match(self._VALID_URL, url) | |
77 | if mobj is None: | |
78 | raise ExtractorError(u'invalid URL: %s' % url) | |
79 | ||
80 | api_base = 'http://api.justin.tv' | |
81 | paged = False | |
82 | if mobj.group('channelid'): | |
83 | paged = True | |
84 | video_id = mobj.group('channelid') | |
85 | api = api_base + '/channel/archives/%s.json' % video_id | |
86 | elif mobj.group('chapterid'): | |
87 | chapter_id = mobj.group('chapterid') | |
88 | ||
89 | webpage = self._download_webpage(url, chapter_id) | |
90 | m = re.search(r'PP\.archive_id = "([0-9]+)";', webpage) | |
91 | if not m: | |
92 | raise ExtractorError(u'Cannot find archive of a chapter') | |
93 | archive_id = m.group(1) | |
94 | ||
95 | api = api_base + '/broadcast/by_chapter/%s.xml' % chapter_id | |
e26f8712 | 96 | doc = self._download_xml(api, chapter_id, |
79e93125 PH |
97 | note=u'Downloading chapter information', |
98 | errnote=u'Chapter information download failed') | |
79e93125 PH |
99 | for a in doc.findall('.//archive'): |
100 | if archive_id == a.find('./id').text: | |
101 | break | |
102 | else: | |
103 | raise ExtractorError(u'Could not find chapter in chapter information') | |
104 | ||
105 | video_url = a.find('./video_file_url').text | |
106 | video_ext = video_url.rpartition('.')[2] or u'flv' | |
107 | ||
108 | chapter_api_url = u'https://api.twitch.tv/kraken/videos/c' + chapter_id | |
109 | chapter_info_json = self._download_webpage(chapter_api_url, u'c' + chapter_id, | |
110 | note='Downloading chapter metadata', | |
111 | errnote='Download of chapter metadata failed') | |
112 | chapter_info = json.loads(chapter_info_json) | |
113 | ||
114 | bracket_start = int(doc.find('.//bracket_start').text) | |
115 | bracket_end = int(doc.find('.//bracket_end').text) | |
116 | ||
117 | # TODO determine start (and probably fix up file) | |
118 | # youtube-dl -v http://www.twitch.tv/firmbelief/c/1757457 | |
119 | #video_url += u'?start=' + TODO:start_timestamp | |
120 | # bracket_start is 13290, but we want 51670615 | |
121 | self._downloader.report_warning(u'Chapter detected, but we can just download the whole file. ' | |
122 | u'Chapter starts at %s and ends at %s' % (formatSeconds(bracket_start), formatSeconds(bracket_end))) | |
123 | ||
124 | info = { | |
125 | 'id': u'c' + chapter_id, | |
126 | 'url': video_url, | |
127 | 'ext': video_ext, | |
128 | 'title': chapter_info['title'], | |
129 | 'thumbnail': chapter_info['preview'], | |
130 | 'description': chapter_info['description'], | |
131 | 'uploader': chapter_info['channel']['display_name'], | |
132 | 'uploader_id': chapter_info['channel']['name'], | |
133 | } | |
134 | return [info] | |
135 | else: | |
136 | video_id = mobj.group('videoid') | |
137 | api = api_base + '/broadcast/by_archive/%s.json' % video_id | |
138 | ||
139 | self.report_extraction(video_id) | |
140 | ||
141 | info = [] | |
142 | offset = 0 | |
143 | limit = self._JUSTIN_PAGE_LIMIT | |
144 | while True: | |
145 | if paged: | |
146 | self.report_download_page(video_id, offset) | |
147 | page_url = api + ('?offset=%d&limit=%d' % (offset, limit)) | |
148 | page_count, page_info = self._parse_page(page_url, video_id) | |
149 | info.extend(page_info) | |
150 | if not paged or page_count != limit: | |
151 | break | |
152 | offset += limit | |
153 | return info |