]>
Commit | Line | Data |
---|---|---|
3ca3f77f U |
1 | from __future__ import unicode_literals |
2 | ||
3 | import itertools | |
4 | import re | |
5 | ||
6 | from .common import InfoExtractor | |
7 | from ..utils import ( | |
8 | int_or_none, | |
9 | js_to_json, | |
10 | orderedSet, | |
11 | parse_duration, | |
12 | sanitized_Request, | |
13 | str_to_int, | |
14 | ) | |
15 | ||
16 | ||
17 | class XTubeIE(InfoExtractor): | |
18 | _VALID_URL = r'''(?x) | |
19 | (?: | |
20 | xtube:| | |
21 | https?://(?:www\.)?xtube\.com/(?:watch\.php\?.*\bv=|video-watch/(?:embedded/)?(?P<display_id>[^/]+)-) | |
22 | ) | |
23 | (?P<id>[^/?&#]+) | |
24 | ''' | |
25 | ||
26 | _TESTS = [{ | |
27 | # old URL schema | |
28 | 'url': 'http://www.xtube.com/watch.php?v=kVTUy_G222_', | |
29 | 'md5': '092fbdd3cbe292c920ef6fc6a8a9cdab', | |
30 | 'info_dict': { | |
31 | 'id': 'kVTUy_G222_', | |
32 | 'ext': 'mp4', | |
33 | 'title': 'strange erotica', | |
34 | 'description': 'contains:an ET kind of thing', | |
35 | 'uploader': 'greenshowers', | |
36 | 'duration': 450, | |
37 | 'view_count': int, | |
38 | 'comment_count': int, | |
39 | 'age_limit': 18, | |
40 | } | |
41 | }, { | |
42 | # FLV videos with duplicated formats | |
43 | 'url': 'http://www.xtube.com/video-watch/A-Super-Run-Part-1-YT-9299752', | |
44 | 'md5': 'a406963eb349dd43692ec54631efd88b', | |
45 | 'info_dict': { | |
46 | 'id': '9299752', | |
47 | 'display_id': 'A-Super-Run-Part-1-YT', | |
48 | 'ext': 'flv', | |
49 | 'title': 'A Super Run - Part 1 (YT)', | |
50 | 'description': 'md5:4cc3af1aa1b0413289babc88f0d4f616', | |
51 | 'uploader': 'tshirtguy59', | |
52 | 'duration': 579, | |
53 | 'view_count': int, | |
54 | 'comment_count': int, | |
55 | 'age_limit': 18, | |
56 | }, | |
57 | }, { | |
58 | # new URL schema | |
59 | 'url': 'http://www.xtube.com/video-watch/strange-erotica-625837', | |
60 | 'only_matching': True, | |
61 | }, { | |
62 | 'url': 'xtube:625837', | |
63 | 'only_matching': True, | |
64 | }, { | |
65 | 'url': 'xtube:kVTUy_G222_', | |
66 | 'only_matching': True, | |
67 | }, { | |
68 | 'url': 'https://www.xtube.com/video-watch/embedded/milf-tara-and-teen-shared-and-cum-covered-extreme-bukkake-32203482?embedsize=big', | |
69 | 'only_matching': True, | |
70 | }] | |
71 | ||
72 | def _real_extract(self, url): | |
73 | mobj = re.match(self._VALID_URL, url) | |
74 | video_id = mobj.group('id') | |
75 | display_id = mobj.group('display_id') | |
76 | ||
77 | if not display_id: | |
78 | display_id = video_id | |
79 | ||
80 | if video_id.isdigit() and len(video_id) < 11: | |
81 | url_pattern = 'http://www.xtube.com/video-watch/-%s' | |
82 | else: | |
83 | url_pattern = 'http://www.xtube.com/watch.php?v=%s' | |
84 | ||
85 | webpage = self._download_webpage( | |
86 | url_pattern % video_id, display_id, headers={ | |
87 | 'Cookie': 'age_verified=1; cookiesAccepted=1', | |
88 | }) | |
89 | ||
90 | title, thumbnail, duration = [None] * 3 | |
91 | ||
92 | config = self._parse_json(self._search_regex( | |
93 | r'playerConf\s*=\s*({.+?})\s*,\s*\n', webpage, 'config', | |
94 | default='{}'), video_id, transform_source=js_to_json, fatal=False) | |
95 | if config: | |
96 | config = config.get('mainRoll') | |
97 | if isinstance(config, dict): | |
98 | title = config.get('title') | |
99 | thumbnail = config.get('poster') | |
100 | duration = int_or_none(config.get('duration')) | |
101 | sources = config.get('sources') or config.get('format') | |
102 | ||
103 | if not isinstance(sources, dict): | |
104 | sources = self._parse_json(self._search_regex( | |
105 | r'(["\'])?sources\1?\s*:\s*(?P<sources>{.+?}),', | |
106 | webpage, 'sources', group='sources'), video_id, | |
107 | transform_source=js_to_json) | |
108 | ||
109 | formats = [] | |
110 | for format_id, format_url in sources.items(): | |
111 | formats.append({ | |
112 | 'url': format_url, | |
113 | 'format_id': format_id, | |
114 | 'height': int_or_none(format_id), | |
115 | }) | |
116 | self._remove_duplicate_formats(formats) | |
117 | self._sort_formats(formats) | |
118 | ||
119 | if not title: | |
120 | title = self._search_regex( | |
121 | (r'<h1>\s*(?P<title>[^<]+?)\s*</h1>', r'videoTitle\s*:\s*(["\'])(?P<title>.+?)\1'), | |
122 | webpage, 'title', group='title') | |
123 | description = self._og_search_description( | |
124 | webpage, default=None) or self._html_search_meta( | |
125 | 'twitter:description', webpage, default=None) or self._search_regex( | |
126 | r'</h1>\s*<p>([^<]+)', webpage, 'description', fatal=False) | |
127 | uploader = self._search_regex( | |
128 | (r'<input[^>]+name="contentOwnerId"[^>]+value="([^"]+)"', | |
129 | r'<span[^>]+class="nickname"[^>]*>([^<]+)'), | |
130 | webpage, 'uploader', fatal=False) | |
131 | if not duration: | |
132 | duration = parse_duration(self._search_regex( | |
133 | r'<dt>Runtime:?</dt>\s*<dd>([^<]+)</dd>', | |
134 | webpage, 'duration', fatal=False)) | |
135 | view_count = str_to_int(self._search_regex( | |
136 | (r'["\']viewsCount["\'][^>]*>(\d+)\s+views', | |
137 | r'<dt>Views:?</dt>\s*<dd>([\d,\.]+)</dd>'), | |
138 | webpage, 'view count', fatal=False)) | |
139 | comment_count = str_to_int(self._html_search_regex( | |
140 | r'>Comments? \(([\d,\.]+)\)<', | |
141 | webpage, 'comment count', fatal=False)) | |
142 | ||
143 | return { | |
144 | 'id': video_id, | |
145 | 'display_id': display_id, | |
146 | 'title': title, | |
147 | 'description': description, | |
148 | 'thumbnail': thumbnail, | |
149 | 'uploader': uploader, | |
150 | 'duration': duration, | |
151 | 'view_count': view_count, | |
152 | 'comment_count': comment_count, | |
153 | 'age_limit': 18, | |
154 | 'formats': formats, | |
155 | } | |
156 | ||
157 | ||
158 | class XTubeUserIE(InfoExtractor): | |
159 | IE_DESC = 'XTube user profile' | |
160 | _VALID_URL = r'https?://(?:www\.)?xtube\.com/profile/(?P<id>[^/]+-\d+)' | |
161 | _TEST = { | |
162 | 'url': 'http://www.xtube.com/profile/greenshowers-4056496', | |
163 | 'info_dict': { | |
164 | 'id': 'greenshowers-4056496', | |
165 | 'age_limit': 18, | |
166 | }, | |
167 | 'playlist_mincount': 154, | |
168 | } | |
169 | ||
170 | def _real_extract(self, url): | |
171 | user_id = self._match_id(url) | |
172 | ||
173 | entries = [] | |
174 | for pagenum in itertools.count(1): | |
175 | request = sanitized_Request( | |
176 | 'http://www.xtube.com/profile/%s/videos/%d' % (user_id, pagenum), | |
177 | headers={ | |
178 | 'Cookie': 'popunder=4', | |
179 | 'X-Requested-With': 'XMLHttpRequest', | |
180 | 'Referer': url, | |
181 | }) | |
182 | ||
183 | page = self._download_json( | |
184 | request, user_id, 'Downloading videos JSON page %d' % pagenum) | |
185 | ||
186 | html = page.get('html') | |
187 | if not html: | |
188 | break | |
189 | ||
190 | for video_id in orderedSet([video_id for _, video_id in re.findall( | |
191 | r'data-plid=(["\'])(.+?)\1', html)]): | |
192 | entries.append(self.url_result('xtube:%s' % video_id, XTubeIE.ie_key())) | |
193 | ||
194 | page_count = int_or_none(page.get('pageCount')) | |
195 | if not page_count or pagenum == page_count: | |
196 | break | |
197 | ||
198 | playlist = self.playlist_result(entries, user_id) | |
199 | playlist['age_limit'] = 18 | |
200 | return playlist |