3 from __future__
import unicode_literals
14 from .common
import InfoExtractor
, SearchInfoExtractor
15 from ..jsinterp
import JSInterpreter
16 from ..swfinterp
import SWFInterpreter
17 from ..compat
import (
20 compat_urllib_parse_unquote
,
21 compat_urllib_parse_unquote_plus
,
22 compat_urllib_parse_urlencode
,
23 compat_urllib_parse_urlparse
,
32 get_element_by_attribute
,
53 class YoutubeBaseInfoExtractor(InfoExtractor
):
54 """Provide base functions for Youtube extractors"""
55 _LOGIN_URL
= 'https://accounts.google.com/ServiceLogin'
56 _TWOFACTOR_URL
= 'https://accounts.google.com/signin/challenge'
57 _PASSWORD_CHALLENGE_URL
= 'https://accounts.google.com/signin/challenge/sl/password'
58 _NETRC_MACHINE
= 'youtube'
59 # If True it will raise an error if no login info is provided
60 _LOGIN_REQUIRED
= False
62 def _set_language(self
):
64 '.youtube.com', 'PREF', 'f1=50000000&hl=en',
65 # YouTube sets the expire time to about two months
66 expire_time
=time
.time() + 2 * 30 * 24 * 3600)
68 def _ids_to_results(self
, ids
):
70 self
.url_result(vid_id
, 'Youtube', video_id
=vid_id
)
75 Attempt to log in to YouTube.
76 True is returned if successful or skipped.
77 False is returned if login failed.
79 If _LOGIN_REQUIRED is set and no authentication was provided, an error is raised.
81 (username
, password
) = self
._get
_login
_info
()
82 # No authentication to be performed
84 if self
._LOGIN
_REQUIRED
:
85 raise ExtractorError('No login info available, needed for using %s.' % self
.IE_NAME
, expected
=True)
88 login_page
= self
._download
_webpage
(
89 self
._LOGIN
_URL
, None,
90 note
='Downloading login page',
91 errnote
='unable to fetch login page', fatal
=False)
92 if login_page
is False:
95 login_form
= self
._hidden
_inputs
(login_page
)
98 'checkConnection': 'youtube',
103 login_results
= self
._download
_webpage
(
104 self
._PASSWORD
_CHALLENGE
_URL
, None,
105 note
='Logging in', errnote
='unable to log in', fatal
=False,
106 data
=urlencode_postdata(login_form
))
107 if login_results
is False:
110 error_msg
= self
._html
_search
_regex
(
111 r
'<[^>]+id="errormsg_0_Passwd"[^>]*>([^<]+)<',
112 login_results
, 'error message', default
=None)
114 raise ExtractorError('Unable to login: %s' % error_msg
, expected
=True)
116 if re
.search(r
'id="errormsg_0_Passwd"', login_results
) is not None:
117 raise ExtractorError('Please use your account password and a two-factor code instead of an application-specific password.', expected
=True)
120 # TODO add SMS and phone call support - these require making a request and then prompting the user
122 if re
.search(r
'(?i)<form[^>]+id="challenge"', login_results
) is not None:
123 tfa_code
= self
._get
_tfa
_info
('2-step verification code')
126 self
._downloader
.report_warning(
127 'Two-factor authentication required. Provide it either interactively or with --twofactor <code>'
128 '(Note that only TOTP (Google Authenticator App) codes work at this time.)')
131 tfa_code
= remove_start(tfa_code
, 'G-')
133 tfa_form_strs
= self
._form
_hidden
_inputs
('challenge', login_results
)
135 tfa_form_strs
.update({
140 tfa_data
= urlencode_postdata(tfa_form_strs
)
142 tfa_req
= sanitized_Request(self
._TWOFACTOR
_URL
, tfa_data
)
143 tfa_results
= self
._download
_webpage
(
145 note
='Submitting TFA code', errnote
='unable to submit tfa', fatal
=False)
147 if tfa_results
is False:
150 if re
.search(r
'(?i)<form[^>]+id="challenge"', tfa_results
) is not None:
151 self
._downloader
.report_warning('Two-factor code expired or invalid. Please try again, or use a one-use backup code instead.')
153 if re
.search(r
'(?i)<form[^>]+id="gaia_loginform"', tfa_results
) is not None:
154 self
._downloader
.report_warning('unable to log in - did the page structure change?')
156 if re
.search(r
'smsauth-interstitial-reviewsettings', tfa_results
) is not None:
157 self
._downloader
.report_warning('Your Google account has a security notice. Please log in on your web browser, resolve the notice, and try again.')
160 if re
.search(r
'(?i)<form[^>]+id="gaia_loginform"', login_results
) is not None:
161 self
._downloader
.report_warning('unable to log in: bad username or password')
165 def _real_initialize(self
):
166 if self
._downloader
is None:
169 if not self
._login
():
173 class YoutubeEntryListBaseInfoExtractor(YoutubeBaseInfoExtractor
):
174 # Extract entries from page with "Load more" button
175 def _entries(self
, page
, playlist_id
):
176 more_widget_html
= content_html
= page
177 for page_num
in itertools
.count(1):
178 for entry
in self
._process
_page
(content_html
):
181 mobj
= re
.search(r
'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html
)
185 more
= self
._download
_json
(
186 'https://youtube.com/%s' % mobj
.group('more'), playlist_id
,
187 'Downloading page #%s' % page_num
,
188 transform_source
=uppercase_escape
)
189 content_html
= more
['content_html']
190 if not content_html
.strip():
191 # Some webpages show a "Load more" button but they don't
194 more_widget_html
= more
['load_more_widget_html']
197 class YoutubePlaylistBaseInfoExtractor(YoutubeEntryListBaseInfoExtractor
):
198 def _process_page(self
, content
):
199 for video_id
, video_title
in self
.extract_videos_from_page(content
):
200 yield self
.url_result(video_id
, 'Youtube', video_id
, video_title
)
202 def extract_videos_from_page(self
, page
):
205 for mobj
in re
.finditer(self
._VIDEO
_RE
, page
):
206 # The link with index 0 is not the first video of the playlist (not sure if still actual)
207 if 'index' in mobj
.groupdict() and mobj
.group('id') == '0':
209 video_id
= mobj
.group('id')
210 video_title
= unescapeHTML(mobj
.group('title'))
212 video_title
= video_title
.strip()
214 idx
= ids_in_page
.index(video_id
)
215 if video_title
and not titles_in_page
[idx
]:
216 titles_in_page
[idx
] = video_title
218 ids_in_page
.append(video_id
)
219 titles_in_page
.append(video_title
)
220 return zip(ids_in_page
, titles_in_page
)
223 class YoutubePlaylistsBaseInfoExtractor(YoutubeEntryListBaseInfoExtractor
):
224 def _process_page(self
, content
):
225 for playlist_id
in orderedSet(re
.findall(
226 r
'<h3[^>]+class="[^"]*yt-lockup-title[^"]*"[^>]*><a[^>]+href="/?playlist\?list=([0-9A-Za-z-_]{10,})"',
228 yield self
.url_result(
229 'https://www.youtube.com/playlist?list=%s' % playlist_id
, 'YoutubePlaylist')
231 def _real_extract(self
, url
):
232 playlist_id
= self
._match
_id
(url
)
233 webpage
= self
._download
_webpage
(url
, playlist_id
)
234 title
= self
._og
_search
_title
(webpage
, fatal
=False)
235 return self
.playlist_result(self
._entries
(webpage
, playlist_id
), playlist_id
, title
)
238 class YoutubeIE(YoutubeBaseInfoExtractor
):
239 IE_DESC
= 'YouTube.com'
240 _VALID_URL
= r
"""(?x)^
242 (?:https?://|//) # http(s):// or protocol-independent URL
243 (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/|
244 (?:www\.)?deturl\.com/www\.youtube\.com/|
245 (?:www\.)?pwnyoutube\.com/|
246 (?:www\.)?yourepeat\.com/|
247 tube\.majestyc\.net/|
248 youtube\.googleapis\.com/) # the various hostnames, with wildcard subdomains
249 (?:.*?\#/)? # handle anchor (#/) redirect urls
250 (?: # the various things that can precede the ID:
251 (?:(?:v|embed|e)/(?!videoseries)) # v/ or embed/ or e/
252 |(?: # or the v= param in all its forms
253 (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)? # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
254 (?:\?|\#!?) # the params delimiter ? or # or #!
255 (?:.*?[&;])?? # any other preceding param (like /?s=tuff&v=xxxx or ?s=tuff&v=V36LpHqtcDY)
260 youtu\.be| # just youtu.be/xxxx
261 vid\.plus| # or vid.plus/xxxx
262 zwearz\.com/watch| # or zwearz.com/watch/xxxx
264 |(?:www\.)?cleanvideosearch\.com/media/action/yt/watch\?videoId=
266 )? # all until now is optional -> you can pass the naked ID
267 ([0-9A-Za-z_-]{11}) # here is it! the YouTube video ID
268 (?!.*?\blist=) # combined list/video URLs are handled by the playlist IE
269 (?(1).+)? # if we found the ID, everything can follow
271 _NEXT_URL_RE
= r
'[\?&]next_url=([^&]+)'
273 '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'}
,
274 '6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'}
,
275 '13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v'}
,
276 '17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'abr': 24, 'vcodec': 'mp4v'}
,
277 '18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 96, 'vcodec': 'h264'}
,
278 '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'}
,
279 '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'}
,
280 '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'}
,
281 # itag 36 videos are either 320x180 (BaW_jenozKc) or 320x240 (__2ABJjxzNo), abr varies as well
282 '36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v'}
,
283 '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'}
,
284 '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'}
,
285 '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'}
,
286 '44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'}
,
287 '45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'}
,
288 '46': {'ext': 'webm', 'width': 1920, 'height': 1080, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'}
,
289 '59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'}
,
290 '78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'}
,
294 '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20}
,
295 '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20}
,
296 '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20}
,
297 '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20}
,
298 '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'preference': -20}
,
299 '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20}
,
300 '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20}
,
302 # Apple HTTP Live Streaming
303 '91': {'ext': 'mp4', 'height': 144, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10}
,
304 '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10}
,
305 '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10}
,
306 '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10}
,
307 '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10}
,
308 '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10}
,
309 '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10}
,
310 '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10}
,
313 '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40}
,
314 '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40}
,
315 '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40}
,
316 '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40}
,
317 '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40}
,
318 '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40}
, # Height can vary (https://github.com/rg3/youtube-dl/issues/4559)
319 '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40}
,
320 '212': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40}
,
321 '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40}
,
322 '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60, 'preference': -40}
,
323 '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60, 'preference': -40}
,
324 '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40}
,
327 '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'preference': -50, 'container': 'm4a_dash'}
,
328 '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'preference': -50, 'container': 'm4a_dash'}
,
329 '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'preference': -50, 'container': 'm4a_dash'}
,
330 '256': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'preference': -50, 'container': 'm4a_dash'}
,
331 '258': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'preference': -50, 'container': 'm4a_dash'}
,
332 '325': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'dtse', 'preference': -50, 'container': 'm4a_dash'}
,
333 '328': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'ec-3', 'preference': -50, 'container': 'm4a_dash'}
,
336 '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40}
,
337 '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40}
,
338 '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40}
,
339 '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40}
,
340 '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40}
,
341 '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40}
,
342 '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp9', 'preference': -40}
,
343 '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40}
,
344 '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40}
,
345 '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40}
,
346 '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40}
,
347 '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40}
,
348 '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40}
,
349 '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40}
,
350 '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40}
,
351 # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
352 '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40}
,
353 '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60, 'preference': -40}
,
354 '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60, 'preference': -40}
,
355 '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60, 'preference': -40}
,
356 '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40}
,
357 '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60, 'preference': -40}
,
360 '171': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 128, 'preference': -50}
,
361 '172': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 256, 'preference': -50}
,
363 # Dash webm audio with opus inside
364 '249': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50, 'preference': -50}
,
365 '250': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70, 'preference': -50}
,
366 '251': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160, 'preference': -50}
,
369 '_rtmp': {'protocol': 'rtmp'}
,
371 _SUBTITLE_FORMATS
= ('ttml', 'vtt')
378 'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&t=1s&end=9',
382 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
383 'uploader': 'Philipp Hagemeister',
384 'uploader_id': 'phihag',
385 'uploader_url': r
're:https?://(?:www\.)?youtube\.com/user/phihag',
386 'upload_date': '20121002',
387 'license': 'Standard YouTube License',
388 'description': 'test chars: "\'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
389 'categories': ['Science & Technology'],
390 'tags': ['youtube-dl'],
393 'dislike_count': int,
399 'url': 'https://www.youtube.com/watch?v=UxxajLWwzqY',
400 'note': 'Test generic use_cipher_signature video (#897)',
404 'upload_date': '20120506',
405 'title': 'Icona Pop - I Love It (feat. Charli XCX) [OFFICIAL VIDEO]',
406 'alt_title': 'I Love It (feat. Charli XCX)',
407 'description': 'md5:f3ceb5ef83a08d95b9d146f973157cc8',
408 'tags': ['Icona Pop i love it', 'sweden', 'pop music', 'big beat records', 'big beat', 'charli',
409 'xcx', 'charli xcx', 'girls', 'hbo', 'i love it', "i don't care", 'icona', 'pop',
410 'iconic ep', 'iconic', 'love', 'it'],
412 'uploader': 'Icona Pop',
413 'uploader_id': 'IconaPop',
414 'uploader_url': r
're:https?://(?:www\.)?youtube\.com/user/IconaPop',
415 'license': 'Standard YouTube License',
416 'creator': 'Icona Pop',
420 'url': 'https://www.youtube.com/watch?v=07FYdnEawAQ',
421 'note': 'Test VEVO video with age protection (#956)',
425 'upload_date': '20130703',
426 'title': 'Justin Timberlake - Tunnel Vision (Explicit)',
427 'alt_title': 'Tunnel Vision',
428 'description': 'md5:64249768eec3bc4276236606ea996373',
430 'uploader': 'justintimberlakeVEVO',
431 'uploader_id': 'justintimberlakeVEVO',
432 'uploader_url': r
're:https?://(?:www\.)?youtube\.com/user/justintimberlakeVEVO',
433 'license': 'Standard YouTube License',
434 'creator': 'Justin Timberlake',
439 'url': '//www.YouTube.com/watch?v=yZIXLfi8CZQ',
440 'note': 'Embed-only video (#1746)',
444 'upload_date': '20120608',
445 'title': 'Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012',
446 'description': 'md5:09b78bd971f1e3e289601dfba15ca4f7',
447 'uploader': 'SET India',
448 'uploader_id': 'setindia',
449 'uploader_url': r
're:https?://(?:www\.)?youtube\.com/user/setindia',
450 'license': 'Standard YouTube License',
455 'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&v=UxxajLWwzqY',
456 'note': 'Use the first video ID in the URL',
460 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
461 'uploader': 'Philipp Hagemeister',
462 'uploader_id': 'phihag',
463 'uploader_url': r
're:https?://(?:www\.)?youtube\.com/user/phihag',
464 'upload_date': '20121002',
465 'license': 'Standard YouTube License',
466 'description': 'test chars: "\'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
467 'categories': ['Science & Technology'],
468 'tags': ['youtube-dl'],
471 'dislike_count': int,
474 'skip_download': True,
478 'url': 'https://www.youtube.com/watch?v=a9LDPn-MO4I',
479 'note': '256k DASH audio (format 141) via DASH manifest',
483 'upload_date': '20121002',
484 'uploader_id': '8KVIDEO',
485 'uploader_url': r
're:https?://(?:www\.)?youtube\.com/user/8KVIDEO',
487 'uploader': '8KVIDEO',
488 'license': 'Standard YouTube License',
489 'title': 'UHDTV TEST 8K VIDEO.mp4'
492 'youtube_include_dash_manifest': True,
495 'skip': 'format 141 not served anymore',
497 # DASH manifest with encrypted signature
499 'url': 'https://www.youtube.com/watch?v=IB3lcPjvWLA',
503 'title': 'Afrojack, Spree Wilson - The Spark ft. Spree Wilson',
504 'description': 'md5:12e7067fa6735a77bdcbb58cb1187d2d',
506 'uploader': 'AfrojackVEVO',
507 'uploader_id': 'AfrojackVEVO',
508 'upload_date': '20131011',
509 'license': 'Standard YouTube License',
512 'youtube_include_dash_manifest': True,
513 'format': '141/bestaudio[ext=m4a]',
516 # JS player signature function name containing $
518 'url': 'https://www.youtube.com/watch?v=nfWlot6h_JM',
522 'title': 'Taylor Swift - Shake It Off',
523 'alt_title': 'Shake It Off',
524 'description': 'md5:95f66187cd7c8b2c13eb78e1223b63c3',
526 'uploader': 'TaylorSwiftVEVO',
527 'uploader_id': 'TaylorSwiftVEVO',
528 'upload_date': '20140818',
529 'license': 'Standard YouTube License',
530 'creator': 'Taylor Swift',
533 'youtube_include_dash_manifest': True,
534 'format': '141/bestaudio[ext=m4a]',
539 'url': 'https://www.youtube.com/watch?v=T4XJQO3qol8',
544 'upload_date': '20100909',
545 'uploader': 'The Amazing Atheist',
546 'uploader_id': 'TheAmazingAtheist',
547 'uploader_url': r
're:https?://(?:www\.)?youtube\.com/user/TheAmazingAtheist',
548 'license': 'Standard YouTube License',
549 'title': 'Burning Everyone\'s Koran',
550 'description': 'SUBSCRIBE: http://www.youtube.com/saturninefilms\n\nEven Obama has taken a stand against freedom on this issue: http://www.huffingtonpost.com/2010/09/09/obama-gma-interview-quran_n_710282.html',
553 # Normal age-gate video (No vevo, embed allowed)
555 'url': 'https://youtube.com/watch?v=HtVdAasjOgU',
559 'title': 'The Witcher 3: Wild Hunt - The Sword Of Destiny Trailer',
560 'description': r
're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}',
562 'uploader': 'The Witcher',
563 'uploader_id': 'WitcherGame',
564 'uploader_url': r
're:https?://(?:www\.)?youtube\.com/user/WitcherGame',
565 'upload_date': '20140605',
566 'license': 'Standard YouTube License',
570 # Age-gate video with encrypted signature
572 'url': 'https://www.youtube.com/watch?v=6kLq3WMV1nU',
576 'title': 'Dedication To My Ex (Miss That) (Lyric Video)',
577 'description': 'md5:33765bb339e1b47e7e72b5490139bb41',
579 'uploader': 'LloydVEVO',
580 'uploader_id': 'LloydVEVO',
581 'uploader_url': r
're:https?://(?:www\.)?youtube\.com/user/LloydVEVO',
582 'upload_date': '20110629',
583 'license': 'Standard YouTube License',
587 # video_info is None (https://github.com/rg3/youtube-dl/issues/4421)
589 'url': '__2ABJjxzNo',
594 'upload_date': '20100430',
595 'uploader_id': 'deadmau5',
596 'uploader_url': r
're:https?://(?:www\.)?youtube\.com/user/deadmau5',
597 'creator': 'deadmau5',
598 'description': 'md5:12c56784b8032162bb936a5f76d55360',
599 'uploader': 'deadmau5',
600 'license': 'Standard YouTube License',
601 'title': 'Deadmau5 - Some Chords (HD)',
602 'alt_title': 'Some Chords',
604 'expected_warnings': [
605 'DASH manifest missing',
608 # Olympics (https://github.com/rg3/youtube-dl/issues/4431)
610 'url': 'lqQg6PlCWgI',
615 'upload_date': '20150827',
616 'uploader_id': 'olympic',
617 'uploader_url': r
're:https?://(?:www\.)?youtube\.com/user/olympic',
618 'license': 'Standard YouTube License',
619 'description': 'HO09 - Women - GER-AUS - Hockey - 31 July 2012 - London 2012 Olympic Games',
620 'uploader': 'Olympic',
621 'title': 'Hockey - Women - GER-AUS - London 2012 Olympic Games',
624 'skip_download': 'requires avconv',
629 'url': 'https://www.youtube.com/watch?v=_b-2C3KPAM0',
633 'stretched_ratio': 16 / 9.,
635 'upload_date': '20110310',
636 'uploader_id': 'AllenMeow',
637 'uploader_url': r
're:https?://(?:www\.)?youtube\.com/user/AllenMeow',
638 'description': 'made by Wacom from Korea | 字幕&加油添醋 by TY\'s Allen | 感謝heylisa00cavey1001同學熱情提供梗及翻譯',
640 'license': 'Standard YouTube License',
641 'title': '[A-made] 變態妍字幕版 太妍 我就是這樣的人',
644 # url_encoded_fmt_stream_map is empty string
646 'url': 'qEJwOuvDf7I',
650 'title': 'Обсуждение судебной практики по выборам 14 сентября 2014 года в Санкт-Петербурге',
652 'upload_date': '20150404',
653 'uploader_id': 'spbelect',
654 'uploader': 'Наблюдатели Петербурга',
657 'skip_download': 'requires avconv',
659 'skip': 'This live event has ended.',
661 # Extraction from multiple DASH manifests (https://github.com/rg3/youtube-dl/pull/6097)
663 'url': 'https://www.youtube.com/watch?v=FIl7x6_3R5Y',
667 'title': 'md5:7b81415841e02ecd4313668cde88737a',
668 'description': 'md5:116377fd2963b81ec4ce64b542173306',
670 'upload_date': '20150625',
671 'uploader_id': 'dorappi2000',
672 'uploader_url': r
're:https?://(?:www\.)?youtube\.com/user/dorappi2000',
673 'uploader': 'dorappi2000',
674 'license': 'Standard YouTube License',
675 'formats': 'mincount:32',
678 # DASH manifest with segment_list
680 'url': 'https://www.youtube.com/embed/CsmdDsKjzN8',
681 'md5': '8ce563a1d667b599d21064e982ab9e31',
685 'upload_date': '20150501', # According to '<meta itemprop="datePublished"', but in other places it's 20150510
686 'uploader': 'Airtek',
687 'description': 'Retransmisión en directo de la XVIII media maratón de Zaragoza.',
688 'uploader_id': 'UCzTzUmjXxxacNnL8I3m4LnQ',
689 'license': 'Standard YouTube License',
690 'title': 'Retransmisión XVIII Media maratón Zaragoza 2015',
693 'youtube_include_dash_manifest': True,
694 'format': '135', # bestvideo
696 'skip': 'This live event has ended.',
699 # Multifeed videos (multiple cameras), URL is for Main Camera
700 'url': 'https://www.youtube.com/watch?v=jqWvoWXjCVs',
703 'title': 'teamPGP: Rocket League Noob Stream',
704 'description': 'md5:dc7872fb300e143831327f1bae3af010',
710 'title': 'teamPGP: Rocket League Noob Stream (Main Camera)',
711 'description': 'md5:dc7872fb300e143831327f1bae3af010',
713 'upload_date': '20150721',
714 'uploader': 'Beer Games Beer',
715 'uploader_id': 'beergamesbeer',
716 'uploader_url': r
're:https?://(?:www\.)?youtube\.com/user/beergamesbeer',
717 'license': 'Standard YouTube License',
723 'title': 'teamPGP: Rocket League Noob Stream (kreestuh)',
724 'description': 'md5:dc7872fb300e143831327f1bae3af010',
726 'upload_date': '20150721',
727 'uploader': 'Beer Games Beer',
728 'uploader_id': 'beergamesbeer',
729 'uploader_url': r
're:https?://(?:www\.)?youtube\.com/user/beergamesbeer',
730 'license': 'Standard YouTube License',
736 'title': 'teamPGP: Rocket League Noob Stream (grizzle)',
737 'description': 'md5:dc7872fb300e143831327f1bae3af010',
739 'upload_date': '20150721',
740 'uploader': 'Beer Games Beer',
741 'uploader_id': 'beergamesbeer',
742 'uploader_url': r
're:https?://(?:www\.)?youtube\.com/user/beergamesbeer',
743 'license': 'Standard YouTube License',
749 'title': 'teamPGP: Rocket League Noob Stream (zim)',
750 'description': 'md5:dc7872fb300e143831327f1bae3af010',
752 'upload_date': '20150721',
753 'uploader': 'Beer Games Beer',
754 'uploader_id': 'beergamesbeer',
755 'uploader_url': r
're:https?://(?:www\.)?youtube\.com/user/beergamesbeer',
756 'license': 'Standard YouTube License',
760 'skip_download': True,
764 # Multifeed video with comma in title (see https://github.com/rg3/youtube-dl/issues/8536)
765 'url': 'https://www.youtube.com/watch?v=gVfLd0zydlo',
768 'title': 'DevConf.cz 2016 Day 2 Workshops 1 14:00 - 15:30',
771 'skip': 'Not multifeed anymore',
774 'url': 'https://vid.plus/FlRa-iH7PGw',
775 'only_matching': True,
778 'url': 'https://zwearz.com/watch/9lWxNJF-ufM/electra-woman-dyna-girl-official-trailer-grace-helbig.html',
779 'only_matching': True,
782 # Title with JS-like syntax "};" (see https://github.com/rg3/youtube-dl/issues/7468)
783 # Also tests cut-off URL expansion in video description (see
784 # https://github.com/rg3/youtube-dl/issues/1892,
785 # https://github.com/rg3/youtube-dl/issues/8164)
786 'url': 'https://www.youtube.com/watch?v=lsguqyKfVQg',
790 'title': '{dark walk}; Loki/AC/Dishonored; collab w/Elflover21',
791 'alt_title': 'Dark Walk',
792 'description': 'md5:8085699c11dc3f597ce0410b0dcbb34a',
794 'upload_date': '20151119',
795 'uploader_id': 'IronSoulElf',
796 'uploader_url': r
're:https?://(?:www\.)?youtube\.com/user/IronSoulElf',
797 'uploader': 'IronSoulElf',
798 'license': 'Standard YouTube License',
799 'creator': 'Todd Haberman, Daniel Law Heath & Aaron Kaplan',
802 'skip_download': True,
806 # Tags with '};' (see https://github.com/rg3/youtube-dl/issues/7468)
807 'url': 'https://www.youtube.com/watch?v=Ms7iBXnlUO8',
808 'only_matching': True,
811 # Video with yt:stretch=17:0
812 'url': 'https://www.youtube.com/watch?v=Q39EVAstoRM',
816 'title': 'Clash Of Clans#14 Dicas De Ataque Para CV 4',
817 'description': 'md5:ee18a25c350637c8faff806845bddee9',
818 'upload_date': '20151107',
819 'uploader_id': 'UCCr7TALkRbo3EtFzETQF1LA',
820 'uploader': 'CH GAMER DROID',
823 'skip_download': True,
825 'skip': 'This video does not exist.',
828 # Video licensed under Creative Commons
829 'url': 'https://www.youtube.com/watch?v=M4gD1WSo5mA',
833 'title': 'md5:e41008789470fc2533a3252216f1c1d1',
834 'description': 'md5:a677553cf0840649b731a3024aeff4cc',
836 'upload_date': '20150127',
837 'uploader_id': 'BerkmanCenter',
838 'uploader_url': r
're:https?://(?:www\.)?youtube\.com/user/BerkmanCenter',
839 'uploader': 'The Berkman Klein Center for Internet & Society',
840 'license': 'Creative Commons Attribution license (reuse allowed)',
843 'skip_download': True,
847 # Channel-like uploader_url
848 'url': 'https://www.youtube.com/watch?v=eQcmzGIKrzg',
852 'title': 'Democratic Socialism and Foreign Policy | Bernie Sanders',
853 'description': 'md5:dda0d780d5a6e120758d1711d062a867',
855 'upload_date': '20151119',
856 'uploader': 'Bernie 2016',
857 'uploader_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
858 'uploader_url': r
're:https?://(?:www\.)?youtube\.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
859 'license': 'Creative Commons Attribution license (reuse allowed)',
862 'skip_download': True,
866 'url': 'https://www.youtube.com/watch?feature=player_embedded&amp;v=V36LpHqtcDY',
867 'only_matching': True,
870 # YouTube Red paid video (https://github.com/rg3/youtube-dl/issues/10059)
871 'url': 'https://www.youtube.com/watch?v=i1Ko8UG-Tdo',
872 'only_matching': True,
875 # Rental video preview
876 'url': 'https://www.youtube.com/watch?v=yYr8q0y5Jfg',
880 'title': 'Piku - Trailer',
881 'description': 'md5:c36bd60c3fd6f1954086c083c72092eb',
882 'upload_date': '20150811',
883 'uploader': 'FlixMatrix',
884 'uploader_id': 'FlixMatrixKaravan',
885 'uploader_url': r
're:https?://(?:www\.)?youtube\.com/user/FlixMatrixKaravan',
886 'license': 'Standard YouTube License',
889 'skip_download': True,
893 # YouTube Red video with episode data
894 'url': 'https://www.youtube.com/watch?v=iqKdEhx-dD4',
898 'title': 'Isolation - Mind Field (Ep 1)',
899 'description': 'md5:8013b7ddea787342608f63a13ddc9492',
901 'upload_date': '20170118',
902 'uploader': 'Vsauce',
903 'uploader_id': 'Vsauce',
904 'uploader_url': r
're:https?://(?:www\.)?youtube\.com/user/Vsauce',
905 'license': 'Standard YouTube License',
906 'series': 'Mind Field',
911 'skip_download': True,
913 'expected_warnings': [
914 'Skipping DASH manifest',
919 'url': '1t24XAntNCY',
920 'only_matching': True,
923 # geo restricted to JP
924 'url': 'sJL6WA-aGkQ',
925 'only_matching': True,
929 def __init__(self
, *args
, **kwargs
):
930 super(YoutubeIE
, self
).__init
__(*args
, **kwargs
)
931 self
._player
_cache
= {}
933 def report_video_info_webpage_download(self
, video_id
):
934 """Report attempt to download video info webpage."""
935 self
.to_screen('%s: Downloading video info webpage' % video_id
)
937 def report_information_extraction(self
, video_id
):
938 """Report attempt to extract video information."""
939 self
.to_screen('%s: Extracting video information' % video_id
)
941 def report_unavailable_format(self
, video_id
, format
):
942 """Report extracted video URL."""
943 self
.to_screen('%s: Format %s not available' % (video_id
, format
))
945 def report_rtmp_download(self
):
946 """Indicate the download will use the RTMP protocol."""
947 self
.to_screen('RTMP download detected')
949 def _signature_cache_id(self
, example_sig
):
950 """ Return a string representation of a signature """
951 return '.'.join(compat_str(len(part
)) for part
in example_sig
.split('.'))
953 def _extract_signature_function(self
, video_id
, player_url
, example_sig
):
955 r
'.*?-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player(?:-new)?|/base)?\.(?P<ext>[a-z]+)$',
958 raise ExtractorError('Cannot identify player %r' % player_url
)
959 player_type
= id_m
.group('ext')
960 player_id
= id_m
.group('id')
962 # Read from filesystem cache
963 func_id
= '%s_%s_%s' % (
964 player_type
, player_id
, self
._signature
_cache
_id
(example_sig
))
965 assert os
.path
.basename(func_id
) == func_id
967 cache_spec
= self
._downloader
.cache
.load('youtube-sigfuncs', func_id
)
968 if cache_spec
is not None:
969 return lambda s
: ''.join(s
[i
] for i
in cache_spec
)
972 'Downloading player %s' % player_url
973 if self
._downloader
.params
.get('verbose') else
974 'Downloading %s player %s' % (player_type
, player_id
)
976 if player_type
== 'js':
977 code
= self
._download
_webpage
(
978 player_url
, video_id
,
980 errnote
='Download of %s failed' % player_url
)
981 res
= self
._parse
_sig
_js
(code
)
982 elif player_type
== 'swf':
983 urlh
= self
._request
_webpage
(
984 player_url
, video_id
,
986 errnote
='Download of %s failed' % player_url
)
988 res
= self
._parse
_sig
_swf
(code
)
990 assert False, 'Invalid player type %r' % player_type
992 test_string
= ''.join(map(compat_chr
, range(len(example_sig
))))
993 cache_res
= res(test_string
)
994 cache_spec
= [ord(c
) for c
in cache_res
]
996 self
._downloader
.cache
.store('youtube-sigfuncs', func_id
, cache_spec
)
999 def _print_sig_code(self
, func
, example_sig
):
1000 def gen_sig_code(idxs
):
1001 def _genslice(start
, end
, step
):
1002 starts
= '' if start
== 0 else str(start
)
1003 ends
= (':%d' % (end
+ step
)) if end
+ step
>= 0 else ':'
1004 steps
= '' if step
== 1 else (':%d' % step
)
1005 return 's[%s%s%s]' % (starts
, ends
, steps
)
1008 # Quelch pyflakes warnings - start will be set when step is set
1009 start
= '(Never used)'
1010 for i
, prev
in zip(idxs
[1:], idxs
[:-1]):
1011 if step
is not None:
1012 if i
- prev
== step
:
1014 yield _genslice(start
, prev
, step
)
1017 if i
- prev
in [-1, 1]:
1022 yield 's[%d]' % prev
1026 yield _genslice(start
, i
, step
)
1028 test_string
= ''.join(map(compat_chr
, range(len(example_sig
))))
1029 cache_res
= func(test_string
)
1030 cache_spec
= [ord(c
) for c
in cache_res
]
1031 expr_code
= ' + '.join(gen_sig_code(cache_spec
))
1032 signature_id_tuple
= '(%s)' % (
1033 ', '.join(compat_str(len(p
)) for p
in example_sig
.split('.')))
1034 code
= ('if tuple(len(p) for p in s.split(\'.\')) == %s:\n'
1035 ' return %s\n') % (signature_id_tuple
, expr_code
)
1036 self
.to_screen('Extracted signature function:\n' + code
)
1038 def _parse_sig_js(self
, jscode
):
1039 funcname
= self
._search
_regex
(
1040 (r
'(["\'])signature\
1\s
*,\s
*(?P
<sig
>[a
-zA
-Z0
-9$
]+)\
(',
1041 r'\
.sig\|\|
(?P
<sig
>[a
-zA
-Z0
-9$
]+)\
('),
1042 jscode, 'Initial JS player signature function name
', group='sig
')
1044 jsi = JSInterpreter(jscode)
1045 initial_function = jsi.extract_function(funcname)
1046 return lambda s: initial_function([s])
1048 def _parse_sig_swf(self, file_contents):
1049 swfi = SWFInterpreter(file_contents)
1050 TARGET_CLASSNAME = 'SignatureDecipher
'
1051 searched_class = swfi.extract_class(TARGET_CLASSNAME)
1052 initial_function = swfi.extract_function(searched_class, 'decipher
')
1053 return lambda s: initial_function([s])
1055 def _decrypt_signature(self, s, video_id, player_url, age_gate=False):
1056 """Turn the encrypted s field into a working signature"""
1058 if player_url is None:
1059 raise ExtractorError('Cannot decrypt signature without player_url
')
1061 if player_url.startswith('//'):
1062 player_url = 'https
:' + player_url
1063 elif not re.match(r'https?
://', player_url):
1064 player_url = compat_urlparse.urljoin(
1065 'https
://www
.youtube
.com
', player_url)
1067 player_id = (player_url, self._signature_cache_id(s))
1068 if player_id not in self._player_cache:
1069 func = self._extract_signature_function(
1070 video_id, player_url, s
1072 self._player_cache[player_id] = func
1073 func = self._player_cache[player_id]
1074 if self._downloader.params.get('youtube_print_sig_code
'):
1075 self._print_sig_code(func, s)
1077 except Exception as e:
1078 tb = traceback.format_exc()
1079 raise ExtractorError(
1080 'Signature extraction failed
: ' + tb, cause=e)
1082 def _get_subtitles(self, video_id, webpage):
1084 subs_doc = self._download_xml(
1085 'https
://video
.google
.com
/timedtext?hl
=en
&type=list&v
=%s' % video_id,
1086 video_id, note=False)
1087 except ExtractorError as err:
1088 self._downloader.report_warning('unable to download video subtitles
: %s' % error_to_compat_str(err))
1092 for track in subs_doc.findall('track
'):
1093 lang = track.attrib['lang_code
']
1094 if lang in sub_lang_list:
1097 for ext in self._SUBTITLE_FORMATS:
1098 params = compat_urllib_parse_urlencode({
1102 'name
': track.attrib['name
'].encode('utf
-8'),
1104 sub_formats.append({
1105 'url
': 'https
://www
.youtube
.com
/api
/timedtext?
' + params,
1108 sub_lang_list[lang] = sub_formats
1109 if not sub_lang_list:
1110 self._downloader.report_warning('video doesn
\'t have subtitles
')
1112 return sub_lang_list
1114 def _get_ytplayer_config(self, video_id, webpage):
1116 # User data may contain arbitrary character sequences that may affect
1117 # JSON extraction with regex, e.g. when '};' is contained the second
1118 # regex won't capture the whole JSON
. Yet working around by trying more
1119 # concrete regex first keeping in mind proper quoted string handling
1120 # to be implemented in future that will replace this workaround (see
1121 # https://github.com/rg3/youtube-dl/issues/7468,
1122 # https://github.com/rg3/youtube-dl/pull/7599)
1123 r
';ytplayer\.config\s*=\s*({.+?});ytplayer',
1124 r
';ytplayer\.config\s*=\s*({.+?});',
1126 config
= self
._search
_regex
(
1127 patterns
, webpage
, 'ytplayer.config', default
=None)
1129 return self
._parse
_json
(
1130 uppercase_escape(config
), video_id
, fatal
=False)
1132 def _get_automatic_captions(self
, video_id
, webpage
):
1133 """We need the webpage for getting the captions url, pass it as an
1134 argument to speed up the process."""
1135 self
.to_screen('%s: Looking for automatic captions' % video_id
)
1136 player_config
= self
._get
_ytplayer
_config
(video_id
, webpage
)
1137 err_msg
= 'Couldn\'t find automatic captions for %s' % video_id
1138 if not player_config
:
1139 self
._downloader
.report_warning(err_msg
)
1142 args
= player_config
['args']
1143 caption_url
= args
.get('ttsurl')
1145 timestamp
= args
['timestamp']
1146 # We get the available subtitles
1147 list_params
= compat_urllib_parse_urlencode({
1152 list_url
= caption_url
+ '&' + list_params
1153 caption_list
= self
._download
_xml
(list_url
, video_id
)
1154 original_lang_node
= caption_list
.find('track')
1155 if original_lang_node
is None:
1156 self
._downloader
.report_warning('Video doesn\'t have automatic captions')
1158 original_lang
= original_lang_node
.attrib
['lang_code']
1159 caption_kind
= original_lang_node
.attrib
.get('kind', '')
1162 for lang_node
in caption_list
.findall('target'):
1163 sub_lang
= lang_node
.attrib
['lang_code']
1165 for ext
in self
._SUBTITLE
_FORMATS
:
1166 params
= compat_urllib_parse_urlencode({
1167 'lang': original_lang
,
1171 'kind': caption_kind
,
1173 sub_formats
.append({
1174 'url': caption_url
+ '&' + params
,
1177 sub_lang_list
[sub_lang
] = sub_formats
1178 return sub_lang_list
1180 # Some videos don't provide ttsurl but rather caption_tracks and
1181 # caption_translation_languages (e.g. 20LmZk1hakA)
1182 caption_tracks
= args
['caption_tracks']
1183 caption_translation_languages
= args
['caption_translation_languages']
1184 caption_url
= compat_parse_qs(caption_tracks
.split(',')[0])['u'][0]
1185 parsed_caption_url
= compat_urllib_parse_urlparse(caption_url
)
1186 caption_qs
= compat_parse_qs(parsed_caption_url
.query
)
1189 for lang
in caption_translation_languages
.split(','):
1190 lang_qs
= compat_parse_qs(compat_urllib_parse_unquote_plus(lang
))
1191 sub_lang
= lang_qs
.get('lc', [None])[0]
1195 for ext
in self
._SUBTITLE
_FORMATS
:
1197 'tlang': [sub_lang
],
1200 sub_url
= compat_urlparse
.urlunparse(parsed_caption_url
._replace
(
1201 query
=compat_urllib_parse_urlencode(caption_qs
, True)))
1202 sub_formats
.append({
1206 sub_lang_list
[sub_lang
] = sub_formats
1207 return sub_lang_list
1208 # An extractor error can be raise by the download process if there are
1209 # no automatic captions but there are subtitles
1210 except (KeyError, ExtractorError
):
1211 self
._downloader
.report_warning(err_msg
)
1214 def _mark_watched(self
, video_id
, video_info
):
1215 playback_url
= video_info
.get('videostats_playback_base_url', [None])[0]
1216 if not playback_url
:
1218 parsed_playback_url
= compat_urlparse
.urlparse(playback_url
)
1219 qs
= compat_urlparse
.parse_qs(parsed_playback_url
.query
)
1221 # cpn generation algorithm is reverse engineered from base.js.
1222 # In fact it works even with dummy cpn.
1223 CPN_ALPHABET
= 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_'
1224 cpn
= ''.join((CPN_ALPHABET
[random
.randint(0, 256) & 63] for _
in range(0, 16)))
1230 playback_url
= compat_urlparse
.urlunparse(
1231 parsed_playback_url
._replace
(query
=compat_urllib_parse_urlencode(qs
, True)))
1233 self
._download
_webpage
(
1234 playback_url
, video_id
, 'Marking watched',
1235 'Unable to mark watched', fatal
=False)
1238 def extract_id(cls
, url
):
1239 mobj
= re
.match(cls
._VALID
_URL
, url
, re
.VERBOSE
)
1241 raise ExtractorError('Invalid URL: %s' % url
)
1242 video_id
= mobj
.group(2)
1245 def _extract_from_m3u8(self
, manifest_url
, video_id
):
1248 def _get_urls(_manifest
):
1249 lines
= _manifest
.split('\n')
1250 urls
= filter(lambda l
: l
and not l
.startswith('#'),
1253 manifest
= self
._download
_webpage
(manifest_url
, video_id
, 'Downloading formats manifest')
1254 formats_urls
= _get_urls(manifest
)
1255 for format_url
in formats_urls
:
1256 itag
= self
._search
_regex
(r
'itag/(\d+?)/', format_url
, 'itag')
1257 url_map
[itag
] = format_url
1260 def _extract_annotations(self
, video_id
):
1261 url
= 'https://www.youtube.com/annotations_invideo?features=1&legacy=1&video_id=%s' % video_id
1262 return self
._download
_webpage
(url
, video_id
, note
='Searching for annotations.', errnote
='Unable to download video annotations.')
1264 def _real_extract(self
, url
):
1265 url
, smuggled_data
= unsmuggle_url(url
, {})
1268 'http' if self
._downloader
.params
.get('prefer_insecure', False)
1273 parsed_url
= compat_urllib_parse_urlparse(url
)
1274 for component
in [parsed_url
.fragment
, parsed_url
.query
]:
1275 query
= compat_parse_qs(component
)
1276 if start_time
is None and 't' in query
:
1277 start_time
= parse_duration(query
['t'][0])
1278 if start_time
is None and 'start' in query
:
1279 start_time
= parse_duration(query
['start'][0])
1280 if end_time
is None and 'end' in query
:
1281 end_time
= parse_duration(query
['end'][0])
1283 # Extract original video URL from URL with redirection, like age verification, using next_url parameter
1284 mobj
= re
.search(self
._NEXT
_URL
_RE
, url
)
1286 url
= proto
+ '://www.youtube.com/' + compat_urllib_parse_unquote(mobj
.group(1)).lstrip('/')
1287 video_id
= self
.extract_id(url
)
1290 url
= proto
+ '://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1&bpctr=9999999999' % video_id
1291 video_webpage
= self
._download
_webpage
(url
, video_id
)
1293 # Attempt to extract SWF player URL
1294 mobj
= re
.search(r
'swfConfig.*?"(https?:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage
)
1295 if mobj
is not None:
1296 player_url
= re
.sub(r
'\\(.)', r
'\1', mobj
.group(1))
1302 def add_dash_mpd(video_info
):
1303 dash_mpd
= video_info
.get('dashmpd')
1304 if dash_mpd
and dash_mpd
[0] not in dash_mpds
:
1305 dash_mpds
.append(dash_mpd
[0])
1308 embed_webpage
= None
1310 if re
.search(r
'player-age-gate-content">', video_webpage
) is not None:
1312 # We simulate the access to the video from www.youtube.com/v/{video_id}
1313 # this can be viewed without login into Youtube
1314 url
= proto
+ '://www.youtube.com/embed/%s' % video_id
1315 embed_webpage
= self
._download
_webpage
(url
, video_id
, 'Downloading embed webpage')
1316 data
= compat_urllib_parse_urlencode({
1317 'video_id': video_id
,
1318 'eurl': 'https://youtube.googleapis.com/v/' + video_id
,
1319 'sts': self
._search
_regex
(
1320 r
'"sts"\s*:\s*(\d+)', embed_webpage
, 'sts', default
=''),
1322 video_info_url
= proto
+ '://www.youtube.com/get_video_info?' + data
1323 video_info_webpage
= self
._download
_webpage
(
1324 video_info_url
, video_id
,
1325 note
='Refetching age-gated info webpage',
1326 errnote
='unable to download video info webpage')
1327 video_info
= compat_parse_qs(video_info_webpage
)
1328 add_dash_mpd(video_info
)
1332 # Try looking directly into the video webpage
1333 ytplayer_config
= self
._get
_ytplayer
_config
(video_id
, video_webpage
)
1335 args
= ytplayer_config
['args']
1336 if args
.get('url_encoded_fmt_stream_map'):
1337 # Convert to the same format returned by compat_parse_qs
1338 video_info
= dict((k
, [v
]) for k
, v
in args
.items())
1339 add_dash_mpd(video_info
)
1340 # Rental video is not rented but preview is available (e.g.
1341 # https://www.youtube.com/watch?v=yYr8q0y5Jfg,
1342 # https://github.com/rg3/youtube-dl/issues/10532)
1343 if not video_info
and args
.get('ypc_vid'):
1344 return self
.url_result(
1345 args
['ypc_vid'], YoutubeIE
.ie_key(), video_id
=args
['ypc_vid'])
1346 if args
.get('livestream') == '1' or args
.get('live_playback') == 1:
1348 if not video_info
or self
._downloader
.params
.get('youtube_include_dash_manifest', True):
1349 # We also try looking in get_video_info since it may contain different dashmpd
1350 # URL that points to a DASH manifest with possibly different itag set (some itags
1351 # are missing from DASH manifest pointed by webpage's dashmpd, some - from DASH
1352 # manifest pointed by get_video_info's dashmpd).
1353 # The general idea is to take a union of itags of both DASH manifests (for example
1354 # video with such 'manifest behavior' see https://github.com/rg3/youtube-dl/issues/6093)
1355 self
.report_video_info_webpage_download(video_id
)
1356 for el_type
in ['&el=info', '&el=embedded', '&el=detailpage', '&el=vevo', '']:
1358 '%s://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
1359 % (proto
, video_id
, el_type
))
1360 video_info_webpage
= self
._download
_webpage
(
1362 video_id
, note
=False,
1363 errnote
='unable to download video info webpage')
1364 get_video_info
= compat_parse_qs(video_info_webpage
)
1365 if get_video_info
.get('use_cipher_signature') != ['True']:
1366 add_dash_mpd(get_video_info
)
1368 video_info
= get_video_info
1369 if 'token' in get_video_info
:
1370 # Different get_video_info requests may report different results, e.g.
1371 # some may report video unavailability, but some may serve it without
1372 # any complaint (see https://github.com/rg3/youtube-dl/issues/7362,
1373 # the original webpage as well as el=info and el=embedded get_video_info
1374 # requests report video unavailability due to geo restriction while
1375 # el=detailpage succeeds and returns valid data). This is probably
1376 # due to YouTube measures against IP ranges of hosting providers.
1377 # Working around by preferring the first succeeded video_info containing
1378 # the token if no such video_info yet was found.
1379 if 'token' not in video_info
:
1380 video_info
= get_video_info
1382 if 'token' not in video_info
:
1383 if 'reason' in video_info
:
1384 if 'The uploader has not made this video available in your country.' in video_info
['reason']:
1385 regions_allowed
= self
._html
_search
_meta
(
1386 'regionsAllowed', video_webpage
, default
=None)
1387 countries
= regions_allowed
.split(',') if regions_allowed
else None
1388 self
.raise_geo_restricted(
1389 msg
=video_info
['reason'][0], countries
=countries
)
1390 raise ExtractorError(
1391 'YouTube said: %s' % video_info
['reason'][0],
1392 expected
=True, video_id
=video_id
)
1394 raise ExtractorError(
1395 '"token" parameter not in video info for unknown reason',
1399 if 'title' in video_info
:
1400 video_title
= video_info
['title'][0]
1402 self
._downloader
.report_warning('Unable to extract video title')
1406 video_description
= get_element_by_id("eow-description", video_webpage
)
1407 if video_description
:
1408 video_description
= re
.sub(r
'''(?x)
1410 (?:[a-zA-Z-]+="[^"]*"\s+)*?
1411 (?:title|href)="([^"]+)"\s+
1412 (?:[a-zA-Z-]+="[^"]*"\s+)*?
1416 ''', r
'\1', video_description
)
1417 video_description
= clean_html(video_description
)
1419 fd_mobj
= re
.search(r
'<meta name="description" content="([^"]+)"', video_webpage
)
1421 video_description
= unescapeHTML(fd_mobj
.group(1))
1423 video_description
= ''
1425 if 'multifeed_metadata_list' in video_info
and not smuggled_data
.get('force_singlefeed', False):
1426 if not self
._downloader
.params
.get('noplaylist'):
1429 multifeed_metadata_list
= video_info
['multifeed_metadata_list'][0]
1430 for feed
in multifeed_metadata_list
.split(','):
1431 # Unquote should take place before split on comma (,) since textual
1432 # fields may contain comma as well (see
1433 # https://github.com/rg3/youtube-dl/issues/8536)
1434 feed_data
= compat_parse_qs(compat_urllib_parse_unquote_plus(feed
))
1436 '_type': 'url_transparent',
1437 'ie_key': 'Youtube',
1439 '%s://www.youtube.com/watch?v=%s' % (proto
, feed_data
['id'][0]),
1440 {'force_singlefeed': True}
),
1441 'title': '%s (%s)' % (video_title
, feed_data
['title'][0]),
1443 feed_ids
.append(feed_data
['id'][0])
1445 'Downloading multifeed video (%s) - add --no-playlist to just download video %s'
1446 % (', '.join(feed_ids
), video_id
))
1447 return self
.playlist_result(entries
, video_id
, video_title
, video_description
)
1448 self
.to_screen('Downloading just video %s because of --no-playlist' % video_id
)
1450 if 'view_count' in video_info
:
1451 view_count
= int(video_info
['view_count'][0])
1455 # Check for "rental" videos
1456 if 'ypc_video_rental_bar_text' in video_info
and 'author' not in video_info
:
1457 raise ExtractorError('"rental" videos not supported')
1459 # Start extracting information
1460 self
.report_information_extraction(video_id
)
1463 if 'author' not in video_info
:
1464 raise ExtractorError('Unable to extract uploader name')
1465 video_uploader
= compat_urllib_parse_unquote_plus(video_info
['author'][0])
1468 video_uploader_id
= None
1469 video_uploader_url
= None
1471 r
'<link itemprop="url" href="(?P<uploader_url>https?://www.youtube.com/(?:user|channel)/(?P<uploader_id>[^"]+))">',
1473 if mobj
is not None:
1474 video_uploader_id
= mobj
.group('uploader_id')
1475 video_uploader_url
= mobj
.group('uploader_url')
1477 self
._downloader
.report_warning('unable to extract uploader nickname')
1480 # We try first to get a high quality image:
1481 m_thumb
= re
.search(r
'<span itemprop="thumbnail".*?href="(.*?)">',
1482 video_webpage
, re
.DOTALL
)
1483 if m_thumb
is not None:
1484 video_thumbnail
= m_thumb
.group(1)
1485 elif 'thumbnail_url' not in video_info
:
1486 self
._downloader
.report_warning('unable to extract video thumbnail')
1487 video_thumbnail
= None
1488 else: # don't panic if we can't find it
1489 video_thumbnail
= compat_urllib_parse_unquote_plus(video_info
['thumbnail_url'][0])
1492 upload_date
= self
._html
_search
_meta
(
1493 'datePublished', video_webpage
, 'upload date', default
=None)
1495 upload_date
= self
._search
_regex
(
1496 [r
'(?s)id="eow-date.*?>(.*?)</span>',
1497 r
'id="watch-uploader-info".*?>.*?(?:Published|Uploaded|Streamed live|Started) on (.+?)</strong>'],
1498 video_webpage
, 'upload date', default
=None)
1500 upload_date
= ' '.join(re
.sub(r
'[/,-]', r
' ', mobj
.group(1)).split())
1501 upload_date
= unified_strdate(upload_date
)
1503 video_license
= self
._html
_search
_regex
(
1504 r
'<h4[^>]+class="title"[^>]*>\s*License\s*</h4>\s*<ul[^>]*>\s*<li>(.+?)</li',
1505 video_webpage
, 'license', default
=None)
1507 m_music
= re
.search(
1508 r
'<h4[^>]+class="title"[^>]*>\s*Music\s*</h4>\s*<ul[^>]*>\s*<li>(?P<title>.+?) by (?P<creator>.+?)(?:\(.+?\))?</li',
1511 video_alt_title
= remove_quotes(unescapeHTML(m_music
.group('title')))
1512 video_creator
= clean_html(m_music
.group('creator'))
1514 video_alt_title
= video_creator
= None
1516 m_episode
= re
.search(
1517 r
'<div[^>]+id="watch7-headline"[^>]*>\s*<span[^>]*>.*?>(?P<series>[^<]+)</a></b>\s*S(?P<season>\d+)\s*•\s*E(?P<episode>\d+)</span>',
1520 series
= m_episode
.group('series')
1521 season_number
= int(m_episode
.group('season'))
1522 episode_number
= int(m_episode
.group('episode'))
1524 series
= season_number
= episode_number
= None
1526 m_cat_container
= self
._search
_regex
(
1527 r
'(?s)<h4[^>]*>\s*Category\s*</h4>\s*<ul[^>]*>(.*?)</ul>',
1528 video_webpage
, 'categories', default
=None)
1530 category
= self
._html
_search
_regex
(
1531 r
'(?s)<a[^<]+>(.*?)</a>', m_cat_container
, 'category',
1533 video_categories
= None if category
is None else [category
]
1535 video_categories
= None
1538 unescapeHTML(m
.group('content'))
1539 for m
in re
.finditer(self
._meta
_regex
('og:video:tag'), video_webpage
)]
1541 def _extract_count(count_name
):
1542 return str_to_int(self
._search
_regex
(
1543 r
'-%s-button[^>]+><span[^>]+class="yt-uix-button-content"[^>]*>([\d,]+)</span>'
1544 % re
.escape(count_name
),
1545 video_webpage
, count_name
, default
=None))
1547 like_count
= _extract_count('like')
1548 dislike_count
= _extract_count('dislike')
1551 video_subtitles
= self
.extract_subtitles(video_id
, video_webpage
)
1552 automatic_captions
= self
.extract_automatic_captions(video_id
, video_webpage
)
1554 video_duration
= try_get(
1555 video_info
, lambda x
: int_or_none(x
['length_seconds'][0]))
1556 if not video_duration
:
1557 video_duration
= parse_duration(self
._html
_search
_meta
(
1558 'duration', video_webpage
, 'video duration'))
1561 video_annotations
= None
1562 if self
._downloader
.params
.get('writeannotations', False):
1563 video_annotations
= self
._extract
_annotations
(video_id
)
1565 def _map_to_format_list(urlmap
):
1567 for itag
, video_real_url
in urlmap
.items():
1570 'url': video_real_url
,
1571 'player_url': player_url
,
1573 if itag
in self
._formats
:
1574 dct
.update(self
._formats
[itag
])
1578 if 'conn' in video_info
and video_info
['conn'][0].startswith('rtmp'):
1579 self
.report_rtmp_download()
1581 'format_id': '_rtmp',
1583 'url': video_info
['conn'][0],
1584 'player_url': player_url
,
1586 elif len(video_info
.get('url_encoded_fmt_stream_map', [''])[0]) >= 1 or len(video_info
.get('adaptive_fmts', [''])[0]) >= 1:
1587 encoded_url_map
= video_info
.get('url_encoded_fmt_stream_map', [''])[0] + ',' + video_info
.get('adaptive_fmts', [''])[0]
1588 if 'rtmpe%3Dyes' in encoded_url_map
:
1589 raise ExtractorError('rtmpe downloads are not supported, see https://github.com/rg3/youtube-dl/issues/343 for more information.', expected
=True)
1591 fmt_list
= video_info
.get('fmt_list', [''])[0]
1593 for fmt
in fmt_list
.split(','):
1594 spec
= fmt
.split('/')
1596 width_height
= spec
[1].split('x')
1597 if len(width_height
) == 2:
1598 formats_spec
[spec
[0]] = {
1599 'resolution': spec
[1],
1600 'width': int_or_none(width_height
[0]),
1601 'height': int_or_none(width_height
[1]),
1604 for url_data_str
in encoded_url_map
.split(','):
1605 url_data
= compat_parse_qs(url_data_str
)
1606 if 'itag' not in url_data
or 'url' not in url_data
:
1608 format_id
= url_data
['itag'][0]
1609 url
= url_data
['url'][0]
1611 if 'sig' in url_data
:
1612 url
+= '&signature=' + url_data
['sig'][0]
1613 elif 's' in url_data
:
1614 encrypted_sig
= url_data
['s'][0]
1615 ASSETS_RE
= r
'"assets":.+?"js":\s*("[^"]+")'
1617 jsplayer_url_json
= self
._search
_regex
(
1619 embed_webpage
if age_gate
else video_webpage
,
1620 'JS player URL (1)', default
=None)
1621 if not jsplayer_url_json
and not age_gate
:
1622 # We need the embed website after all
1623 if embed_webpage
is None:
1624 embed_url
= proto
+ '://www.youtube.com/embed/%s' % video_id
1625 embed_webpage
= self
._download
_webpage
(
1626 embed_url
, video_id
, 'Downloading embed webpage')
1627 jsplayer_url_json
= self
._search
_regex
(
1628 ASSETS_RE
, embed_webpage
, 'JS player URL')
1630 player_url
= json
.loads(jsplayer_url_json
)
1631 if player_url
is None:
1632 player_url_json
= self
._search
_regex
(
1633 r
'ytplayer\.config.*?"url"\s*:\s*("[^"]+")',
1634 video_webpage
, 'age gate player URL')
1635 player_url
= json
.loads(player_url_json
)
1637 if self
._downloader
.params
.get('verbose'):
1638 if player_url
is None:
1639 player_version
= 'unknown'
1640 player_desc
= 'unknown'
1642 if player_url
.endswith('swf'):
1643 player_version
= self
._search
_regex
(
1644 r
'-(.+?)(?:/watch_as3)?\.swf$', player_url
,
1645 'flash player', fatal
=False)
1646 player_desc
= 'flash player %s' % player_version
1648 player_version
= self
._search
_regex
(
1649 [r
'html5player-([^/]+?)(?:/html5player(?:-new)?)?\.js', r
'(?:www|player)-([^/]+)/base\.js'],
1651 'html5 player', fatal
=False)
1652 player_desc
= 'html5 player %s' % player_version
1654 parts_sizes
= self
._signature
_cache
_id
(encrypted_sig
)
1655 self
.to_screen('{%s} signature length %s, %s' %
1656 (format_id
, parts_sizes
, player_desc
))
1658 signature
= self
._decrypt
_signature
(
1659 encrypted_sig
, video_id
, player_url
, age_gate
)
1660 url
+= '&signature=' + signature
1661 if 'ratebypass' not in url
:
1662 url
+= '&ratebypass=yes'
1665 'format_id': format_id
,
1667 'player_url': player_url
,
1669 if format_id
in self
._formats
:
1670 dct
.update(self
._formats
[format_id
])
1671 if format_id
in formats_spec
:
1672 dct
.update(formats_spec
[format_id
])
1674 # Some itags are not included in DASH manifest thus corresponding formats will
1675 # lack metadata (see https://github.com/rg3/youtube-dl/pull/5993).
1676 # Trying to extract metadata from url_encoded_fmt_stream_map entry.
1677 mobj
= re
.search(r
'^(?P<width>\d+)[xX](?P<height>\d+)$', url_data
.get('size', [''])[0])
1678 width
, height
= (int(mobj
.group('width')), int(mobj
.group('height'))) if mobj
else (None, None)
1681 'filesize': int_or_none(url_data
.get('clen', [None])[0]),
1682 'tbr': float_or_none(url_data
.get('bitrate', [None])[0], 1000),
1685 'fps': int_or_none(url_data
.get('fps', [None])[0]),
1686 'format_note': url_data
.get('quality_label', [None])[0] or url_data
.get('quality', [None])[0],
1688 for key
, value
in more_fields
.items():
1691 type_
= url_data
.get('type', [None])[0]
1693 type_split
= type_
.split(';')
1694 kind_ext
= type_split
[0].split('/')
1695 if len(kind_ext
) == 2:
1697 dct
['ext'] = mimetype2ext(type_split
[0])
1698 if kind
in ('audio', 'video'):
1700 for mobj
in re
.finditer(
1701 r
'(?P<key>[a-zA-Z_-]+)=(?P<quote>["\']?
)(?P
<val
>.+?
)(?P
=quote
)(?
:;|$
)', type_):
1702 if mobj.group('key
') == 'codecs
':
1703 codecs = mobj.group('val
')
1706 dct.update(parse_codecs(codecs))
1708 elif video_info.get('hlsvp
'):
1709 manifest_url = video_info['hlsvp
'][0]
1710 url_map = self._extract_from_m3u8(manifest_url, video_id)
1711 formats = _map_to_format_list(url_map)
1712 # Accept-Encoding header causes failures in live streams on Youtube and Youtube Gaming
1713 for a_format in formats:
1714 a_format.setdefault('http_headers
', {})['Youtubedl
-no
-compression
'] = 'True'
1716 unavailable_message = self._html_search_regex(
1717 r'(?s
)<h1
[^
>]+id="unavailable-message"[^
>]*>(.+?
)</h1
>',
1718 video_webpage, 'unavailable message
', default=None)
1719 if unavailable_message:
1720 raise ExtractorError(unavailable_message, expected=True)
1721 raise ExtractorError('no conn
, hlsvp
or url_encoded_fmt_stream_map information found
in video info
')
1723 # Look for the DASH manifest
1724 if self._downloader.params.get('youtube_include_dash_manifest
', True):
1725 dash_mpd_fatal = True
1726 for mpd_url in dash_mpds:
1729 def decrypt_sig(mobj):
1731 dec_s = self._decrypt_signature(s, video_id, player_url, age_gate)
1732 return '/signature
/%s' % dec_s
1734 mpd_url = re.sub(r'/s
/([a
-fA
-F0
-9\
.]+)', decrypt_sig, mpd_url)
1736 for df in self._extract_mpd_formats(
1737 mpd_url, video_id, fatal=dash_mpd_fatal,
1738 formats_dict=self._formats):
1739 # Do not overwrite DASH format found in some previous DASH manifest
1740 if df['format_id
'] not in dash_formats:
1741 dash_formats[df['format_id
']] = df
1742 # Additional DASH manifests may end up in HTTP Error 403 therefore
1743 # allow them to fail without bug report message if we already have
1744 # some DASH manifest succeeded. This is temporary workaround to reduce
1745 # burst of bug reports until we figure out the reason and whether it
1746 # can be fixed at all.
1747 dash_mpd_fatal = False
1748 except (ExtractorError, KeyError) as e:
1749 self.report_warning(
1750 'Skipping DASH manifest
: %r' % e, video_id)
1752 # Remove the formats we found through non-DASH, they
1753 # contain less info and it can be wrong, because we use
1754 # fixed values (for example the resolution). See
1755 # https://github.com/rg3/youtube-dl/issues/5774 for an
1757 formats = [f for f in formats if f['format_id
'] not in dash_formats.keys()]
1758 formats.extend(dash_formats.values())
1760 # Check for malformed aspect ratio
1761 stretched_m = re.search(
1762 r'<meta\s
+property="og:video:tag".*?content
="yt:stretch=(?P<w>[0-9]+):(?P<h>[0-9]+)">',
1765 w = float(stretched_m.group('w
'))
1766 h = float(stretched_m.group('h
'))
1767 # yt:stretch may hold invalid ratio data (e.g. for Q39EVAstoRM ratio is 17:0).
1768 # We will only process correct ratios.
1772 if f.get('vcodec
') != 'none
':
1773 f['stretched_ratio
'] = ratio
1775 self._sort_formats(formats)
1777 self.mark_watched(video_id, video_info)
1781 'uploader
': video_uploader,
1782 'uploader_id
': video_uploader_id,
1783 'uploader_url
': video_uploader_url,
1784 'upload_date
': upload_date,
1785 'license
': video_license,
1786 'creator
': video_creator,
1787 'title
': video_title,
1788 'alt_title
': video_alt_title,
1789 'thumbnail
': video_thumbnail,
1790 'description
': video_description,
1791 'categories
': video_categories,
1793 'subtitles
': video_subtitles,
1794 'automatic_captions
': automatic_captions,
1795 'duration
': video_duration,
1796 'age_limit
': 18 if age_gate else 0,
1797 'annotations
': video_annotations,
1798 'webpage_url
': proto + '://www
.youtube
.com
/watch?v
=%s' % video_id,
1799 'view_count
': view_count,
1800 'like_count
': like_count,
1801 'dislike_count
': dislike_count,
1802 'average_rating
': float_or_none(video_info.get('avg_rating
', [None])[0]),
1805 'start_time
': start_time,
1806 'end_time
': end_time,
1808 'season_number
': season_number,
1809 'episode_number
': episode_number,
1813 class YoutubeSharedVideoIE(InfoExtractor):
1814 _VALID_URL = r'(?
:https?
:)?
//(?
:www\
.)?youtube\
.com
/shared
\?.*\bci
=(?P
<id>[0-9A
-Za
-z_
-]{11}
)'
1815 IE_NAME = 'youtube
:shared
'
1818 'url
': 'https
://www
.youtube
.com
/shared?ci
=1nEzmT
-M4fU
',
1820 'id': 'uPDB5I9wfp8
',
1822 'title
': 'Pocoyo
: 90 minutos de episódios completos Português para crianç
as - PARTE
3',
1823 'description
': 'md5
:d9e4d9346a2dfff4c7dc4c8cec0f546d
',
1824 'upload_date
': '20160219',
1825 'uploader
': 'Pocoyo
- Portuguê
s (BR
)',
1826 'uploader_id
': 'PocoyoBrazil
',
1828 'add_ie
': ['Youtube
'],
1830 # There are already too many Youtube downloads
1831 'skip_download
': True,
1835 def _real_extract(self, url):
1836 video_id = self._match_id(url)
1838 webpage = self._download_webpage(url, video_id)
1840 real_video_id = self._html_search_meta(
1841 'videoId
', webpage, 'YouTube video
id', fatal=True)
1843 return self.url_result(real_video_id, YoutubeIE.ie_key())
1846 class YoutubePlaylistIE(YoutubePlaylistBaseInfoExtractor):
1847 IE_DESC = 'YouTube
.com playlists
'
1848 _VALID_URL = r"""(?x)(?:
1854 (?:course|view_play_list|my_playlists|artist|playlist|watch|embed/videoseries)
1855 \? (?:.*?[&;])*? (?:p|a|list)=
1858 youtu\.be/[0-9A-Za-z_-]{11}\?.*?\blist=
1861 (?:PL|LL|EC|UU|FL|RD|UL|TL)?[0-9A-Za-z-_]{10,}
1862 # Top tracks, they can also include dots
1867 ((?:PL|LL|EC|UU|FL|RD|UL|TL)[0-9A-Za-z-_]{10,})
1869 _TEMPLATE_URL = 'https
://www
.youtube
.com
/playlist?
list=%s&disable_polymer
=true
'
1870 _VIDEO_RE = r'href
="\s*/watch\?v=(?P<id>[0-9A-Za-z_-]{11})&[^"]*?index
=(?P
<index
>\d
+)(?
:[^
>]+>(?P
<title
>[^
<]+))?
'
1871 IE_NAME = 'youtube
:playlist
'
1873 'url
': 'https
://www
.youtube
.com
/playlist?
list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re
',
1875 'title
': 'ytdl test PL
',
1876 'id': 'PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re
',
1878 'playlist_count
': 3,
1880 'url
': 'https
://www
.youtube
.com
/playlist?
list=PLtPgu7CB4gbZDA7i_euNxn75ISqxwZPYx
',
1882 'id': 'PLtPgu7CB4gbZDA7i_euNxn75ISqxwZPYx
',
1883 'title
': 'YDL_Empty_List
',
1885 'playlist_count
': 0,
1886 'skip
': 'This playlist
is private
',
1888 'note
': 'Playlist
with deleted
videos (#651). As a bonus, the video #51 is also twice in this list.',
1889 'url': 'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
1891 'title': '29C3: Not my department',
1892 'id': 'PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
1894 'playlist_count': 95,
1896 'note': 'issue #673',
1897 'url': 'PLBB231211A4F62143',
1899 'title': '[OLD]Team Fortress 2 (Class-based LP)',
1900 'id': 'PLBB231211A4F62143',
1902 'playlist_mincount': 26,
1904 'note': 'Large playlist',
1905 'url': 'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q',
1907 'title': 'Uploads from Cauchemar',
1908 'id': 'UUBABnxM4Ar9ten8Mdjj1j0Q',
1910 'playlist_mincount': 799,
1912 'url': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
1914 'title': 'YDL_safe_search',
1915 'id': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
1917 'playlist_count': 2,
1918 'skip': 'This playlist is private',
1921 'url': 'https://www.youtube.com/embed/videoseries?list=PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
1922 'playlist_count': 4,
1925 'id': 'PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
1928 'note': 'Embedded SWF player',
1929 'url': 'https://www.youtube.com/p/YN5VISEtHet5D4NEvfTd0zcgFk84NqFZ?hl=en_US&fs=1&rel=0',
1930 'playlist_count': 4,
1933 'id': 'YN5VISEtHet5D4NEvfTd0zcgFk84NqFZ',
1936 'note': 'Buggy playlist: the webpage has a "Load more" button but it doesn\'t have more videos',
1937 'url': 'https://www.youtube.com/playlist?list=UUXw-G3eDE9trcvY2sBMM_aA',
1939 'title': 'Uploads from Interstellar Movie',
1940 'id': 'UUXw-G3eDE9trcvY2sBMM_aA',
1942 'playlist_mincount': 21,
1944 # Playlist URL that does not actually serve a playlist
1945 'url': 'https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4',
1947 'id': 'FqZTN594JQw',
1949 'title': "Smiley's People 01 detective, Adventure Series, Action",
1950 'uploader': 'STREEM',
1951 'uploader_id': 'UCyPhqAZgwYWZfxElWVbVJng',
1952 'uploader_url': r
're:https?://(?:www\.)?youtube\.com/channel/UCyPhqAZgwYWZfxElWVbVJng',
1953 'upload_date': '20150526',
1954 'license': 'Standard YouTube License',
1955 'description': 'md5:507cdcb5a49ac0da37a920ece610be80',
1956 'categories': ['People & Blogs'],
1959 'dislike_count': int,
1962 'skip_download': True,
1964 'add_ie': [YoutubeIE
.ie_key()],
1966 'url': 'https://youtu.be/yeWKywCrFtk?list=PL2qgrgXsNUG5ig9cat4ohreBjYLAPC0J5',
1968 'id': 'yeWKywCrFtk',
1970 'title': 'Small Scale Baler and Braiding Rugs',
1971 'uploader': 'Backus-Page House Museum',
1972 'uploader_id': 'backuspagemuseum',
1973 'uploader_url': r
're:https?://(?:www\.)?youtube\.com/user/backuspagemuseum',
1974 'upload_date': '20161008',
1975 'license': 'Standard YouTube License',
1976 'description': 'md5:800c0c78d5eb128500bffd4f0b4f2e8a',
1977 'categories': ['Nonprofits & Activism'],
1980 'dislike_count': int,
1984 'skip_download': True,
1987 'url': 'https://youtu.be/uWyaPkt-VOI?list=PL9D9FC436B881BA21',
1988 'only_matching': True,
1990 'url': 'TLGGrESM50VT6acwMjAyMjAxNw',
1991 'only_matching': True,
1994 def _real_initialize(self
):
1997 def _extract_mix(self
, playlist_id
):
1998 # The mixes are generated from a single video
1999 # the id of the playlist is just 'RD' + video_id
2001 last_id
= playlist_id
[-11:]
2002 for n
in itertools
.count(1):
2003 url
= 'https://youtube.com/watch?v=%s&list=%s' % (last_id
, playlist_id
)
2004 webpage
= self
._download
_webpage
(
2005 url
, playlist_id
, 'Downloading page {0} of Youtube mix'.format(n
))
2006 new_ids
= orderedSet(re
.findall(
2007 r
'''(?xs)data-video-username=".*?".*?
2008 href="/watch\?v=([0-9A-Za-z_-]{11})&[^"]*?list=%s''' % re
.escape(playlist_id
),
2010 # Fetch new pages until all the videos are repeated, it seems that
2011 # there are always 51 unique videos.
2012 new_ids
= [_id
for _id
in new_ids
if _id
not in ids
]
2018 url_results
= self
._ids
_to
_results
(ids
)
2020 search_title
= lambda class_name
: get_element_by_attribute('class', class_name
, webpage
)
2022 search_title('playlist-title') or
2023 search_title('title long-title') or
2024 search_title('title'))
2025 title
= clean_html(title_span
)
2027 return self
.playlist_result(url_results
, playlist_id
, title
)
2029 def _extract_playlist(self
, playlist_id
):
2030 url
= self
._TEMPLATE
_URL
% playlist_id
2031 page
= self
._download
_webpage
(url
, playlist_id
)
2033 # the yt-alert-message now has tabindex attribute (see https://github.com/rg3/youtube-dl/issues/11604)
2034 for match
in re
.findall(r
'<div class="yt-alert-message"[^>]*>([^<]+)</div>', page
):
2035 match
= match
.strip()
2036 # Check if the playlist exists or is private
2037 mobj
= re
.match(r
'[^<]*(?:The|This) playlist (?P<reason>does not exist|is private)[^<]*', match
)
2039 reason
= mobj
.group('reason')
2040 message
= 'This playlist %s' % reason
2041 if 'private' in reason
:
2042 message
+= ', use --username or --netrc to access it'
2044 raise ExtractorError(message
, expected
=True)
2045 elif re
.match(r
'[^<]*Invalid parameters[^<]*', match
):
2046 raise ExtractorError(
2047 'Invalid parameters. Maybe URL is incorrect.',
2049 elif re
.match(r
'[^<]*Choose your language[^<]*', match
):
2052 self
.report_warning('Youtube gives an alert message: ' + match
)
2054 playlist_title
= self
._html
_search
_regex
(
2055 r
'(?s)<h1 class="pl-header-title[^"]*"[^>]*>\s*(.*?)\s*</h1>',
2056 page
, 'title', default
=None)
2060 if not playlist_title
:
2062 # Some playlist URLs don't actually serve a playlist (e.g.
2063 # https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4)
2064 next(self
._entries
(page
, playlist_id
))
2065 except StopIteration:
2068 return has_videos
, self
.playlist_result(
2069 self
._entries
(page
, playlist_id
), playlist_id
, playlist_title
)
2071 def _check_download_just_video(self
, url
, playlist_id
):
2072 # Check if it's a video-specific URL
2073 query_dict
= compat_urlparse
.parse_qs(compat_urlparse
.urlparse(url
).query
)
2074 video_id
= query_dict
.get('v', [None])[0] or self
._search
_regex
(
2075 r
'(?:^|//)youtu\.be/([0-9A-Za-z_-]{11})', url
,
2076 'video id', default
=None)
2078 if self
._downloader
.params
.get('noplaylist'):
2079 self
.to_screen('Downloading just video %s because of --no-playlist' % video_id
)
2080 return video_id
, self
.url_result(video_id
, 'Youtube', video_id
=video_id
)
2082 self
.to_screen('Downloading playlist %s - add --no-playlist to just download video %s' % (playlist_id
, video_id
))
2083 return video_id
, None
2086 def _real_extract(self
, url
):
2087 # Extract playlist id
2088 mobj
= re
.match(self
._VALID
_URL
, url
)
2090 raise ExtractorError('Invalid URL: %s' % url
)
2091 playlist_id
= mobj
.group(1) or mobj
.group(2)
2093 video_id
, video
= self
._check
_download
_just
_video
(url
, playlist_id
)
2097 if playlist_id
.startswith(('RD', 'UL', 'PU')):
2098 # Mixes require a custom extraction process
2099 return self
._extract
_mix
(playlist_id
)
2101 has_videos
, playlist
= self
._extract
_playlist
(playlist_id
)
2102 if has_videos
or not video_id
:
2105 # Some playlist URLs don't actually serve a playlist (see
2106 # https://github.com/rg3/youtube-dl/issues/10537).
2107 # Fallback to plain video extraction if there is a video id
2108 # along with playlist id.
2109 return self
.url_result(video_id
, 'Youtube', video_id
=video_id
)
2112 class YoutubeChannelIE(YoutubePlaylistBaseInfoExtractor
):
2113 IE_DESC
= 'YouTube.com channels'
2114 _VALID_URL
= r
'https?://(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie)?\.com)/channel/(?P<id>[0-9A-Za-z_-]+)'
2115 _TEMPLATE_URL
= 'https://www.youtube.com/channel/%s/videos'
2116 _VIDEO_RE
= r
'(?:title="(?P<title>[^"]+)"[^>]+)?href="/watch\?v=(?P<id>[0-9A-Za-z_-]+)&?'
2117 IE_NAME
= 'youtube:channel'
2119 'note': 'paginated channel',
2120 'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
2121 'playlist_mincount': 91,
2123 'id': 'UUKfVa3S1e4PHvxWcwyMMg8w',
2124 'title': 'Uploads from lex will',
2127 'note': 'Age restricted channel',
2128 # from https://www.youtube.com/user/DeusExOfficial
2129 'url': 'https://www.youtube.com/channel/UCs0ifCMCm1icqRbqhUINa0w',
2130 'playlist_mincount': 64,
2132 'id': 'UUs0ifCMCm1icqRbqhUINa0w',
2133 'title': 'Uploads from Deus Ex',
2136 # geo restricted to JP
2137 'url': 'https://www.youtube.com/user/kananishinoSMEJ',
2138 'only_matching': True,
2142 def suitable(cls
, url
):
2143 return (False if YoutubePlaylistsIE
.suitable(url
) or YoutubeLiveIE
.suitable(url
)
2144 else super(YoutubeChannelIE
, cls
).suitable(url
))
2146 def _build_template_url(self
, url
, channel_id
):
2147 return self
._TEMPLATE
_URL
% channel_id
2149 def _real_extract(self
, url
):
2150 channel_id
= self
._match
_id
(url
)
2152 url
= self
._build
_template
_url
(url
, channel_id
)
2154 # Channel by page listing is restricted to 35 pages of 30 items, i.e. 1050 videos total (see #5778)
2155 # Workaround by extracting as a playlist if managed to obtain channel playlist URL
2156 # otherwise fallback on channel by page extraction
2157 channel_page
= self
._download
_webpage
(
2158 url
+ '?view=57', channel_id
,
2159 'Downloading channel page', fatal
=False)
2160 if channel_page
is False:
2161 channel_playlist_id
= False
2163 channel_playlist_id
= self
._html
_search
_meta
(
2164 'channelId', channel_page
, 'channel id', default
=None)
2165 if not channel_playlist_id
:
2166 channel_url
= self
._html
_search
_meta
(
2167 ('al:ios:url', 'twitter:app:url:iphone', 'twitter:app:url:ipad'),
2168 channel_page
, 'channel url', default
=None)
2170 channel_playlist_id
= self
._search
_regex
(
2171 r
'vnd\.youtube://user/([0-9A-Za-z_-]+)',
2172 channel_url
, 'channel id', default
=None)
2173 if channel_playlist_id
and channel_playlist_id
.startswith('UC'):
2174 playlist_id
= 'UU' + channel_playlist_id
[2:]
2175 return self
.url_result(
2176 compat_urlparse
.urljoin(url
, '/playlist?list=%s' % playlist_id
), 'YoutubePlaylist')
2178 channel_page
= self
._download
_webpage
(url
, channel_id
, 'Downloading page #1')
2179 autogenerated
= re
.search(r
'''(?x)
2181 channel-header-autogenerated-label|
2182 yt-channel-title-autogenerated
2183 )[^"]*"''', channel_page
) is not None
2186 # The videos are contained in a single page
2187 # the ajax pages can't be used, they are empty
2190 video_id
, 'Youtube', video_id
=video_id
,
2191 video_title
=video_title
)
2192 for video_id
, video_title
in self
.extract_videos_from_page(channel_page
)]
2193 return self
.playlist_result(entries
, channel_id
)
2196 next(self
._entries
(channel_page
, channel_id
))
2197 except StopIteration:
2198 alert_message
= self
._html
_search
_regex
(
2199 r
'(?s)<div[^>]+class=(["\']).*?
\byt
-alert
-message
\b.*?\
1[^
>]*>(?P
<alert
>[^
<]+)</div
>',
2200 channel_page, 'alert
', default=None, group='alert
')
2202 raise ExtractorError('Youtube said
: %s' % alert_message, expected=True)
2204 return self.playlist_result(self._entries(channel_page, channel_id), channel_id)
2207 class YoutubeUserIE(YoutubeChannelIE):
2208 IE_DESC = 'YouTube
.com user
videos (URL
or "ytuser" keyword
)'
2209 _VALID_URL = r'(?
:(?
:https?
://(?
:\w
+\
.)?youtube\
.com
/(?
:(?P
<user
>user|c
)/)?
(?
!(?
:attribution_link|watch|results
)(?
:$|
[^a
-z_A
-Z0
-9-])))|ytuser
:)(?
!feed
/)(?P
<id>[A
-Za
-z0
-9_-]+)'
2210 _TEMPLATE_URL = 'https
://www
.youtube
.com
/%s/%s/videos
'
2211 IE_NAME = 'youtube
:user
'
2214 'url
': 'https
://www
.youtube
.com
/user
/TheLinuxFoundation
',
2215 'playlist_mincount
': 320,
2217 'id': 'UUfX55Sx5hEFjoC3cNs6mCUQ
',
2218 'title
': 'Uploads
from The Linux Foundation
',
2221 # Only available via https://www.youtube.com/c/12minuteathlete/videos
2222 # but not https://www.youtube.com/user/12minuteathlete/videos
2223 'url
': 'https
://www
.youtube
.com
/c
/12minuteathlete
/videos
',
2224 'playlist_mincount
': 249,
2226 'id': 'UUVjM
-zV6_opMDx7WYxnjZiQ
',
2227 'title
': 'Uploads
from 12 Minute Athlete
',
2230 'url
': 'ytuser
:phihag
',
2231 'only_matching
': True,
2233 'url
': 'https
://www
.youtube
.com
/c
/gametrailers
',
2234 'only_matching
': True,
2236 'url
': 'https
://www
.youtube
.com
/gametrailers
',
2237 'only_matching
': True,
2239 # This channel is not available.
2240 'url
': 'https
://www
.youtube
.com
/user
/kananishinoSMEJ
/videos
',
2241 'only_matching
': True,
2245 def suitable(cls, url):
2246 # Don't
return True if the url can be extracted
with other youtube
2247 # extractor, the regex would is too permissive and it would match.
2248 other_yt_ies
= iter(klass
for (name
, klass
) in globals().items() if name
.startswith('Youtube') and name
.endswith('IE') and klass
is not cls
)
2249 if any(ie
.suitable(url
) for ie
in other_yt_ies
):
2252 return super(YoutubeUserIE
, cls
).suitable(url
)
2254 def _build_template_url(self
, url
, channel_id
):
2255 mobj
= re
.match(self
._VALID
_URL
, url
)
2256 return self
._TEMPLATE
_URL
% (mobj
.group('user') or 'user', mobj
.group('id'))
2259 class YoutubeLiveIE(YoutubeBaseInfoExtractor
):
2260 IE_DESC
= 'YouTube.com live streams'
2261 _VALID_URL
= r
'(?P<base_url>https?://(?:\w+\.)?youtube\.com/(?:(?:user|channel|c)/)?(?P<id>[^/]+))/live'
2262 IE_NAME
= 'youtube:live'
2265 'url': 'https://www.youtube.com/user/TheYoungTurks/live',
2267 'id': 'a48o2S1cPoo',
2269 'title': 'The Young Turks - Live Main Show',
2270 'uploader': 'The Young Turks',
2271 'uploader_id': 'TheYoungTurks',
2272 'uploader_url': r
're:https?://(?:www\.)?youtube\.com/user/TheYoungTurks',
2273 'upload_date': '20150715',
2274 'license': 'Standard YouTube License',
2275 'description': 'md5:438179573adcdff3c97ebb1ee632b891',
2276 'categories': ['News & Politics'],
2277 'tags': ['Cenk Uygur (TV Program Creator)', 'The Young Turks (Award-Winning Work)', 'Talk Show (TV Genre)'],
2279 'dislike_count': int,
2282 'skip_download': True,
2285 'url': 'https://www.youtube.com/channel/UC1yBKRuGpC1tSM73A0ZjYjQ/live',
2286 'only_matching': True,
2288 'url': 'https://www.youtube.com/c/CommanderVideoHq/live',
2289 'only_matching': True,
2291 'url': 'https://www.youtube.com/TheYoungTurks/live',
2292 'only_matching': True,
2295 def _real_extract(self
, url
):
2296 mobj
= re
.match(self
._VALID
_URL
, url
)
2297 channel_id
= mobj
.group('id')
2298 base_url
= mobj
.group('base_url')
2299 webpage
= self
._download
_webpage
(url
, channel_id
, fatal
=False)
2301 page_type
= self
._og
_search
_property
(
2302 'type', webpage
, 'page type', default
=None)
2303 video_id
= self
._html
_search
_meta
(
2304 'videoId', webpage
, 'video id', default
=None)
2305 if page_type
== 'video' and video_id
and re
.match(r
'^[0-9A-Za-z_-]{11}$', video_id
):
2306 return self
.url_result(video_id
, YoutubeIE
.ie_key())
2307 return self
.url_result(base_url
)
2310 class YoutubePlaylistsIE(YoutubePlaylistsBaseInfoExtractor
):
2311 IE_DESC
= 'YouTube.com user/channel playlists'
2312 _VALID_URL
= r
'https?://(?:\w+\.)?youtube\.com/(?:user|channel)/(?P<id>[^/]+)/playlists'
2313 IE_NAME
= 'youtube:playlists'
2316 'url': 'https://www.youtube.com/user/ThirstForScience/playlists',
2317 'playlist_mincount': 4,
2319 'id': 'ThirstForScience',
2320 'title': 'Thirst for Science',
2323 # with "Load more" button
2324 'url': 'https://www.youtube.com/user/igorkle1/playlists?view=1&sort=dd',
2325 'playlist_mincount': 70,
2328 'title': 'Игорь Клейнер',
2331 'url': 'https://www.youtube.com/channel/UCiU1dHvZObB2iP6xkJ__Icw/playlists',
2332 'playlist_mincount': 17,
2334 'id': 'UCiU1dHvZObB2iP6xkJ__Icw',
2335 'title': 'Chem Player',
2340 class YoutubeSearchIE(SearchInfoExtractor
, YoutubePlaylistIE
):
2341 IE_DESC
= 'YouTube.com searches'
2342 # there doesn't appear to be a real limit, for example if you search for
2343 # 'python' you get more than 8.000.000 results
2344 _MAX_RESULTS
= float('inf')
2345 IE_NAME
= 'youtube:search'
2346 _SEARCH_KEY
= 'ytsearch'
2347 _EXTRA_QUERY_ARGS
= {}
2350 def _get_n_results(self
, query
, n
):
2351 """Get a specified number of results for a query"""
2357 'search_query': query
.encode('utf-8'),
2359 url_query
.update(self
._EXTRA
_QUERY
_ARGS
)
2360 result_url
= 'https://www.youtube.com/results?' + compat_urllib_parse_urlencode(url_query
)
2362 for pagenum
in itertools
.count(1):
2363 data
= self
._download
_json
(
2364 result_url
, video_id
='query "%s"' % query
,
2365 note
='Downloading page %s' % pagenum
,
2366 errnote
='Unable to download API page',
2367 query
={'spf': 'navigate'}
)
2368 html_content
= data
[1]['body']['content']
2370 if 'class="search-message' in html_content
:
2371 raise ExtractorError(
2372 '[youtube] No video results', expected
=True)
2374 new_videos
= self
._ids
_to
_results
(orderedSet(re
.findall(
2375 r
'href="/watch\?v=(.{11})', html_content
)))
2376 videos
+= new_videos
2377 if not new_videos
or len(videos
) > limit
:
2379 next_link
= self
._html
_search
_regex
(
2380 r
'href="(/results\?[^"]*\bsp=[^"]+)"[^>]*>\s*<span[^>]+class="[^"]*\byt-uix-button-content\b[^"]*"[^>]*>Next',
2381 html_content
, 'next link', default
=None)
2382 if next_link
is None:
2384 result_url
= compat_urlparse
.urljoin('https://www.youtube.com/', next_link
)
2388 return self
.playlist_result(videos
, query
)
2391 class YoutubeSearchDateIE(YoutubeSearchIE
):
2392 IE_NAME
= YoutubeSearchIE
.IE_NAME
+ ':date'
2393 _SEARCH_KEY
= 'ytsearchdate'
2394 IE_DESC
= 'YouTube.com searches, newest videos first'
2395 _EXTRA_QUERY_ARGS
= {'search_sort': 'video_date_uploaded'}
2398 class YoutubeSearchURLIE(YoutubePlaylistBaseInfoExtractor
):
2399 IE_DESC
= 'YouTube.com search URLs'
2400 IE_NAME
= 'youtube:search_url'
2401 _VALID_URL
= r
'https?://(?:www\.)?youtube\.com/results\?(.*?&)?(?:search_query|q)=(?P<query>[^&]+)(?:[&]|$)'
2402 _VIDEO_RE
= r
'href="\s*/watch\?v=(?P<id>[0-9A-Za-z_-]{11})(?:[^"]*"[^>]+\btitle="(?P<title>[^"]+))?'
2404 'url': 'https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video',
2405 'playlist_mincount': 5,
2407 'title': 'youtube-dl test video',
2410 'url': 'https://www.youtube.com/results?q=test&sp=EgQIBBgB',
2411 'only_matching': True,
2414 def _real_extract(self
, url
):
2415 mobj
= re
.match(self
._VALID
_URL
, url
)
2416 query
= compat_urllib_parse_unquote_plus(mobj
.group('query'))
2417 webpage
= self
._download
_webpage
(url
, query
)
2418 return self
.playlist_result(self
._process
_page
(webpage
), playlist_title
=query
)
2421 class YoutubeShowIE(YoutubePlaylistsBaseInfoExtractor
):
2422 IE_DESC
= 'YouTube.com (multi-season) shows'
2423 _VALID_URL
= r
'https?://(?:www\.)?youtube\.com/show/(?P<id>[^?#]*)'
2424 IE_NAME
= 'youtube:show'
2426 'url': 'https://www.youtube.com/show/airdisasters',
2427 'playlist_mincount': 5,
2429 'id': 'airdisasters',
2430 'title': 'Air Disasters',
2434 def _real_extract(self
, url
):
2435 playlist_id
= self
._match
_id
(url
)
2436 return super(YoutubeShowIE
, self
)._real
_extract
(
2437 'https://www.youtube.com/show/%s/playlists' % playlist_id
)
2440 class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor
):
2442 Base class for feed extractors
2443 Subclasses must define the _FEED_NAME and _PLAYLIST_TITLE properties.
2445 _LOGIN_REQUIRED
= True
2449 return 'youtube:%s' % self
._FEED
_NAME
2451 def _real_initialize(self
):
2454 def _real_extract(self
, url
):
2455 page
= self
._download
_webpage
(
2456 'https://www.youtube.com/feed/%s' % self
._FEED
_NAME
, self
._PLAYLIST
_TITLE
)
2458 # The extraction process is the same as for playlists, but the regex
2459 # for the video ids doesn't contain an index
2461 more_widget_html
= content_html
= page
2462 for page_num
in itertools
.count(1):
2463 matches
= re
.findall(r
'href="\s*/watch\?v=([0-9A-Za-z_-]{11})', content_html
)
2465 # 'recommended' feed has infinite 'load more' and each new portion spins
2466 # the same videos in (sometimes) slightly different order, so we'll check
2467 # for unicity and break when portion has no new videos
2468 new_ids
= filter(lambda video_id
: video_id
not in ids
, orderedSet(matches
))
2474 mobj
= re
.search(r
'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html
)
2478 more
= self
._download
_json
(
2479 'https://youtube.com/%s' % mobj
.group('more'), self
._PLAYLIST
_TITLE
,
2480 'Downloading page #%s' % page_num
,
2481 transform_source
=uppercase_escape
)
2482 content_html
= more
['content_html']
2483 more_widget_html
= more
['load_more_widget_html']
2485 return self
.playlist_result(
2486 self
._ids
_to
_results
(ids
), playlist_title
=self
._PLAYLIST
_TITLE
)
2489 class YoutubeWatchLaterIE(YoutubePlaylistIE
):
2490 IE_NAME
= 'youtube:watchlater'
2491 IE_DESC
= 'Youtube watch later list, ":ytwatchlater" for short (requires authentication)'
2492 _VALID_URL
= r
'https?://(?:www\.)?youtube\.com/(?:feed/watch_later|(?:playlist|watch)\?(?:.+&)?list=WL)|:ytwatchlater'
2495 'url': 'https://www.youtube.com/playlist?list=WL',
2496 'only_matching': True,
2498 'url': 'https://www.youtube.com/watch?v=bCNU9TrbiRk&index=1&list=WL',
2499 'only_matching': True,
2502 def _real_extract(self
, url
):
2503 _
, video
= self
._check
_download
_just
_video
(url
, 'WL')
2506 _
, playlist
= self
._extract
_playlist
('WL')
2510 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor
):
2511 IE_NAME
= 'youtube:favorites'
2512 IE_DESC
= 'YouTube.com favourite videos, ":ytfav" for short (requires authentication)'
2513 _VALID_URL
= r
'https?://(?:www\.)?youtube\.com/my_favorites|:ytfav(?:ou?rites)?'
2514 _LOGIN_REQUIRED
= True
2516 def _real_extract(self
, url
):
2517 webpage
= self
._download
_webpage
('https://www.youtube.com/my_favorites', 'Youtube Favourites videos')
2518 playlist_id
= self
._search
_regex
(r
'list=(.+?)["&]', webpage
, 'favourites playlist id')
2519 return self
.url_result(playlist_id
, 'YoutubePlaylist')
2522 class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor
):
2523 IE_DESC
= 'YouTube.com recommended videos, ":ytrec" for short (requires authentication)'
2524 _VALID_URL
= r
'https?://(?:www\.)?youtube\.com/feed/recommended|:ytrec(?:ommended)?'
2525 _FEED_NAME
= 'recommended'
2526 _PLAYLIST_TITLE
= 'Youtube Recommended videos'
2529 class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor
):
2530 IE_DESC
= 'YouTube.com subscriptions feed, "ytsubs" keyword (requires authentication)'
2531 _VALID_URL
= r
'https?://(?:www\.)?youtube\.com/feed/subscriptions|:ytsubs(?:criptions)?'
2532 _FEED_NAME
= 'subscriptions'
2533 _PLAYLIST_TITLE
= 'Youtube Subscriptions'
2536 class YoutubeHistoryIE(YoutubeFeedsInfoExtractor
):
2537 IE_DESC
= 'Youtube watch history, ":ythistory" for short (requires authentication)'
2538 _VALID_URL
= r
'https?://(?:www\.)?youtube\.com/feed/history|:ythistory'
2539 _FEED_NAME
= 'history'
2540 _PLAYLIST_TITLE
= 'Youtube History'
2543 class YoutubeTruncatedURLIE(InfoExtractor
):
2544 IE_NAME
= 'youtube:truncated_url'
2545 IE_DESC
= False # Do not list
2546 _VALID_URL
= r
'''(?x)
2548 (?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/
2551 annotation_id=annotation_[^&]+|
2557 attribution_link\?a=[^&]+
2563 'url': 'https://www.youtube.com/watch?annotation_id=annotation_3951667041',
2564 'only_matching': True,
2566 'url': 'https://www.youtube.com/watch?',
2567 'only_matching': True,
2569 'url': 'https://www.youtube.com/watch?x-yt-cl=84503534',
2570 'only_matching': True,
2572 'url': 'https://www.youtube.com/watch?feature=foo',
2573 'only_matching': True,
2575 'url': 'https://www.youtube.com/watch?hl=en-GB',
2576 'only_matching': True,
2578 'url': 'https://www.youtube.com/watch?t=2372',
2579 'only_matching': True,
2582 def _real_extract(self
, url
):
2583 raise ExtractorError(
2584 'Did you forget to quote the URL? Remember that & is a meta '
2585 'character in most shells, so you want to put the URL in quotes, '
2587 '"https://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
2588 ' or simply youtube-dl BaW_jenozKc .',
2592 class YoutubeTruncatedIDIE(InfoExtractor
):
2593 IE_NAME
= 'youtube:truncated_id'
2594 IE_DESC
= False # Do not list
2595 _VALID_URL
= r
'https?://(?:www\.)?youtube\.com/watch\?v=(?P<id>[0-9A-Za-z_-]{1,10})$'
2598 'url': 'https://www.youtube.com/watch?v=N_708QY7Ob',
2599 'only_matching': True,
2602 def _real_extract(self
, url
):
2603 video_id
= self
._match
_id
(url
)
2604 raise ExtractorError(
2605 'Incomplete YouTube ID %s. URL %s looks truncated.' % (video_id
, url
),