]>
Commit | Line | Data |
---|---|---|
79e93125 PH |
1 | import json |
2 | import os | |
3 | import re | |
4 | import xml.etree.ElementTree | |
5 | ||
6 | from .common import InfoExtractor | |
7 | from ..utils import ( | |
8 | ExtractorError, | |
9 | formatSeconds, | |
10 | ) | |
11 | ||
12 | ||
13 | class JustinTVIE(InfoExtractor): | |
14 | """Information extractor for justin.tv and twitch.tv""" | |
15 | # TODO: One broadcast may be split into multiple videos. The key | |
16 | # 'broadcast_id' is the same for all parts, and 'broadcast_part' | |
17 | # starts at 1 and increases. Can we treat all parts as one video? | |
18 | ||
19 | _VALID_URL = r"""(?x)^(?:http://)?(?:www\.)?(?:twitch|justin)\.tv/ | |
20 | (?: | |
21 | (?P<channelid>[^/]+)| | |
22 | (?:(?:[^/]+)/b/(?P<videoid>[^/]+))| | |
23 | (?:(?:[^/]+)/c/(?P<chapterid>[^/]+)) | |
24 | ) | |
25 | /?(?:\#.*)?$ | |
26 | """ | |
27 | _JUSTIN_PAGE_LIMIT = 100 | |
28 | IE_NAME = u'justin.tv' | |
6f5ac90c PH |
29 | _TEST = { |
30 | u'url': u'http://www.twitch.tv/thegamedevhub/b/296128360', | |
31 | u'file': u'296128360.flv', | |
32 | u'md5': u'ecaa8a790c22a40770901460af191c9a', | |
33 | u'info_dict': { | |
34 | u"upload_date": u"20110927", | |
35 | u"uploader_id": 25114803, | |
36 | u"uploader": u"thegamedevhub", | |
37 | u"title": u"Beginner Series - Scripting With Python Pt.1" | |
38 | } | |
39 | } | |
79e93125 PH |
40 | |
41 | def report_download_page(self, channel, offset): | |
42 | """Report attempt to download a single page of videos.""" | |
43 | self.to_screen(u'%s: Downloading video information from %d to %d' % | |
44 | (channel, offset, offset + self._JUSTIN_PAGE_LIMIT)) | |
45 | ||
46 | # Return count of items, list of *valid* items | |
47 | def _parse_page(self, url, video_id): | |
190717e3 PH |
48 | info_json = self._download_webpage(url, video_id, |
49 | u'Downloading video info JSON', | |
50 | u'unable to download video info JSON') | |
79e93125 | 51 | |
190717e3 | 52 | response = json.loads(info_json) |
79e93125 PH |
53 | if type(response) != list: |
54 | error_text = response.get('error', 'unknown error') | |
55 | raise ExtractorError(u'Justin.tv API: %s' % error_text) | |
56 | info = [] | |
57 | for clip in response: | |
58 | video_url = clip['video_file_url'] | |
59 | if video_url: | |
60 | video_extension = os.path.splitext(video_url)[1][1:] | |
61 | video_date = re.sub('-', '', clip['start_time'][:10]) | |
62 | video_uploader_id = clip.get('user_id', clip.get('channel_id')) | |
63 | video_id = clip['id'] | |
64 | video_title = clip.get('title', video_id) | |
65 | info.append({ | |
66 | 'id': video_id, | |
67 | 'url': video_url, | |
68 | 'title': video_title, | |
69 | 'uploader': clip.get('channel_name', video_uploader_id), | |
70 | 'uploader_id': video_uploader_id, | |
71 | 'upload_date': video_date, | |
72 | 'ext': video_extension, | |
73 | }) | |
74 | return (len(response), info) | |
75 | ||
76 | def _real_extract(self, url): | |
77 | mobj = re.match(self._VALID_URL, url) | |
78 | if mobj is None: | |
79 | raise ExtractorError(u'invalid URL: %s' % url) | |
80 | ||
81 | api_base = 'http://api.justin.tv' | |
82 | paged = False | |
83 | if mobj.group('channelid'): | |
84 | paged = True | |
85 | video_id = mobj.group('channelid') | |
86 | api = api_base + '/channel/archives/%s.json' % video_id | |
87 | elif mobj.group('chapterid'): | |
88 | chapter_id = mobj.group('chapterid') | |
89 | ||
90 | webpage = self._download_webpage(url, chapter_id) | |
91 | m = re.search(r'PP\.archive_id = "([0-9]+)";', webpage) | |
92 | if not m: | |
93 | raise ExtractorError(u'Cannot find archive of a chapter') | |
94 | archive_id = m.group(1) | |
95 | ||
96 | api = api_base + '/broadcast/by_chapter/%s.xml' % chapter_id | |
97 | chapter_info_xml = self._download_webpage(api, chapter_id, | |
98 | note=u'Downloading chapter information', | |
99 | errnote=u'Chapter information download failed') | |
100 | doc = xml.etree.ElementTree.fromstring(chapter_info_xml) | |
101 | for a in doc.findall('.//archive'): | |
102 | if archive_id == a.find('./id').text: | |
103 | break | |
104 | else: | |
105 | raise ExtractorError(u'Could not find chapter in chapter information') | |
106 | ||
107 | video_url = a.find('./video_file_url').text | |
108 | video_ext = video_url.rpartition('.')[2] or u'flv' | |
109 | ||
110 | chapter_api_url = u'https://api.twitch.tv/kraken/videos/c' + chapter_id | |
111 | chapter_info_json = self._download_webpage(chapter_api_url, u'c' + chapter_id, | |
112 | note='Downloading chapter metadata', | |
113 | errnote='Download of chapter metadata failed') | |
114 | chapter_info = json.loads(chapter_info_json) | |
115 | ||
116 | bracket_start = int(doc.find('.//bracket_start').text) | |
117 | bracket_end = int(doc.find('.//bracket_end').text) | |
118 | ||
119 | # TODO determine start (and probably fix up file) | |
120 | # youtube-dl -v http://www.twitch.tv/firmbelief/c/1757457 | |
121 | #video_url += u'?start=' + TODO:start_timestamp | |
122 | # bracket_start is 13290, but we want 51670615 | |
123 | self._downloader.report_warning(u'Chapter detected, but we can just download the whole file. ' | |
124 | u'Chapter starts at %s and ends at %s' % (formatSeconds(bracket_start), formatSeconds(bracket_end))) | |
125 | ||
126 | info = { | |
127 | 'id': u'c' + chapter_id, | |
128 | 'url': video_url, | |
129 | 'ext': video_ext, | |
130 | 'title': chapter_info['title'], | |
131 | 'thumbnail': chapter_info['preview'], | |
132 | 'description': chapter_info['description'], | |
133 | 'uploader': chapter_info['channel']['display_name'], | |
134 | 'uploader_id': chapter_info['channel']['name'], | |
135 | } | |
136 | return [info] | |
137 | else: | |
138 | video_id = mobj.group('videoid') | |
139 | api = api_base + '/broadcast/by_archive/%s.json' % video_id | |
140 | ||
141 | self.report_extraction(video_id) | |
142 | ||
143 | info = [] | |
144 | offset = 0 | |
145 | limit = self._JUSTIN_PAGE_LIMIT | |
146 | while True: | |
147 | if paged: | |
148 | self.report_download_page(video_id, offset) | |
149 | page_url = api + ('?offset=%d&limit=%d' % (offset, limit)) | |
150 | page_count, page_info = self._parse_page(page_url, video_id) | |
151 | info.extend(page_info) | |
152 | if not paged or page_count != limit: | |
153 | break | |
154 | offset += limit | |
155 | return info |