]> jfr.im git - yt-dlp.git/blob - youtube_dl/extractor/youtube.py
projects / yt-dlp.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Do not warn if fallback is without alternatives (because we did not get the flash...
[yt-dlp.git] / youtube_dl / extractor / youtube.py
1 # coding: utf-8
2
3 import collections
4 import errno
5 import io
6 import itertools
7 import json
8 import os.path
9 import re
10 import socket
11 import string
12 import struct
13 import traceback
14 import xml.etree.ElementTree
15 import zlib
16
17 from .common import InfoExtractor, SearchInfoExtractor
18 from .subtitles import SubtitlesInfoExtractor
19 from ..utils import (
20 compat_chr,
21 compat_http_client,
22 compat_parse_qs,
23 compat_urllib_error,
24 compat_urllib_parse,
25 compat_urllib_request,
26 compat_str,
27
28 clean_html,
29 get_element_by_id,
30 ExtractorError,
31 unescapeHTML,
32 unified_strdate,
33 orderedSet,
34 write_json_file,
35 )
36
37 class YoutubeBaseInfoExtractor(InfoExtractor):
38 """Provide base functions for Youtube extractors"""
39 _LOGIN_URL = 'https://accounts.google.com/ServiceLogin'
40 _LANG_URL = r'https://www.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
41 _AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
42 _NETRC_MACHINE = 'youtube'
43 # If True it will raise an error if no login info is provided
44 _LOGIN_REQUIRED = False
45
46 def report_lang(self):
47 """Report attempt to set language."""
48 self.to_screen(u'Setting language')
49
50 def _set_language(self):
51 request = compat_urllib_request.Request(self._LANG_URL)
52 try:
53 self.report_lang()
54 compat_urllib_request.urlopen(request).read()
55 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
56 self._downloader.report_warning(u'unable to set language: %s' % compat_str(err))
57 return False
58 return True
59
60 def _login(self):
61 (username, password) = self._get_login_info()
62 # No authentication to be performed
63 if username is None:
64 if self._LOGIN_REQUIRED:
65 raise ExtractorError(u'No login info available, needed for using %s.' % self.IE_NAME, expected=True)
66 return False
67
68 request = compat_urllib_request.Request(self._LOGIN_URL)
69 try:
70 login_page = compat_urllib_request.urlopen(request).read().decode('utf-8')
71 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
72 self._downloader.report_warning(u'unable to fetch login page: %s' % compat_str(err))
73 return False
74
75 galx = None
76 dsh = None
77 match = re.search(re.compile(r'<input.+?name="GALX".+?value="(.+?)"', re.DOTALL), login_page)
78 if match:
79 galx = match.group(1)
80 match = re.search(re.compile(r'<input.+?name="dsh".+?value="(.+?)"', re.DOTALL), login_page)
81 if match:
82 dsh = match.group(1)
83
84 # Log in
85 login_form_strs = {
86 u'continue': u'https://www.youtube.com/signin?action_handle_signin=true&feature=sign_in_button&hl=en_US&nomobiletemp=1',
87 u'Email': username,
88 u'GALX': galx,
89 u'Passwd': password,
90 u'PersistentCookie': u'yes',
91 u'_utf8': u'霱',
92 u'bgresponse': u'js_disabled',
93 u'checkConnection': u'',
94 u'checkedDomains': u'youtube',
95 u'dnConn': u'',
96 u'dsh': dsh,
97 u'pstMsg': u'0',
98 u'rmShown': u'1',
99 u'secTok': u'',
100 u'signIn': u'Sign in',
101 u'timeStmp': u'',
102 u'service': u'youtube',
103 u'uilel': u'3',
104 u'hl': u'en_US',
105 }
106 # Convert to UTF-8 *before* urlencode because Python 2.x's urlencode
107 # chokes on unicode
108 login_form = dict((k.encode('utf-8'), v.encode('utf-8')) for k,v in login_form_strs.items())
109 login_data = compat_urllib_parse.urlencode(login_form).encode('ascii')
110 request = compat_urllib_request.Request(self._LOGIN_URL, login_data)
111 try:
112 self.report_login()
113 login_results = compat_urllib_request.urlopen(request).read().decode('utf-8')
114 if re.search(r'(?i)<form[^>]* id="gaia_loginform"', login_results) is not None:
115 self._downloader.report_warning(u'unable to log in: bad username or password')
116 return False
117 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
118 self._downloader.report_warning(u'unable to log in: %s' % compat_str(err))
119 return False
120 return True
121
122 def _confirm_age(self):
123 age_form = {
124 'next_url': '/',
125 'action_confirm': 'Confirm',
126 }
127 request = compat_urllib_request.Request(self._AGE_URL, compat_urllib_parse.urlencode(age_form))
128 try:
129 self.report_age_confirmation()
130 compat_urllib_request.urlopen(request).read().decode('utf-8')
131 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
132 raise ExtractorError(u'Unable to confirm age: %s' % compat_str(err))
133 return True
134
135 def _real_initialize(self):
136 if self._downloader is None:
137 return
138 if not self._set_language():
139 return
140 if not self._login():
141 return
142 self._confirm_age()
143
144
145 class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
146 IE_DESC = u'YouTube.com'
147 _VALID_URL = r"""^
148 (
149 (?:https?://)? # http(s):// (optional)
150 (?:(?:(?:(?:\w+\.)?youtube(?:-nocookie)?\.com/|
151 tube\.majestyc\.net/|
152 youtube\.googleapis\.com/) # the various hostnames, with wildcard subdomains
153 (?:.*?\#/)? # handle anchor (#/) redirect urls
154 (?: # the various things that can precede the ID:
155 (?:(?:v|embed|e)/) # v/ or embed/ or e/
156 |(?: # or the v= param in all its forms
157 (?:(?:watch|movie)(?:_popup)?(?:\.php)?)? # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
158 (?:\?|\#!?) # the params delimiter ? or # or #!
159 (?:.*?&)? # any other preceding param (like /?s=tuff&v=xxxx)
160 v=
161 )
162 ))
163 |youtu\.be/ # just youtu.be/xxxx
164 )
165 )? # all until now is optional -> you can pass the naked ID
166 ([0-9A-Za-z_-]{11}) # here is it! the YouTube video ID
167 (?(1).+)? # if we found the ID, everything can follow
168 $"""
169 _NEXT_URL_RE = r'[\?&]next_url=([^&]+)'
170 # Listed in order of quality
171 _available_formats = ['38', '37', '46', '22', '45', '35', '44', '34', '18', '43', '6', '5', '36', '17', '13',
172 # Apple HTTP Live Streaming
173 '96', '95', '94', '93', '92', '132', '151',
174 # 3D
175 '85', '84', '102', '83', '101', '82', '100',
176 # Dash video
177 '138', '137', '248', '136', '247', '135', '246',
178 '245', '244', '134', '243', '133', '242', '160',
179 # Dash audio
180 '141', '172', '140', '171', '139',
181 ]
182 _available_formats_prefer_free = ['38', '46', '37', '45', '22', '44', '35', '43', '34', '18', '6', '5', '36', '17', '13',
183 # Apple HTTP Live Streaming
184 '96', '95', '94', '93', '92', '132', '151',
185 # 3D
186 '85', '102', '84', '101', '83', '100', '82',
187 # Dash video
188 '138', '248', '137', '247', '136', '246', '245',
189 '244', '135', '243', '134', '242', '133', '160',
190 # Dash audio
191 '172', '141', '171', '140', '139',
192 ]
193 _video_formats_map = {
194 'flv': ['35', '34', '6', '5'],
195 '3gp': ['36', '17', '13'],
196 'mp4': ['38', '37', '22', '18'],
197 'webm': ['46', '45', '44', '43'],
198 }
199 _video_extensions = {
200 '13': '3gp',
201 '17': '3gp',
202 '18': 'mp4',
203 '22': 'mp4',
204 '36': '3gp',
205 '37': 'mp4',
206 '38': 'mp4',
207 '43': 'webm',
208 '44': 'webm',
209 '45': 'webm',
210 '46': 'webm',
211
212 # 3d videos
213 '82': 'mp4',
214 '83': 'mp4',
215 '84': 'mp4',
216 '85': 'mp4',
217 '100': 'webm',
218 '101': 'webm',
219 '102': 'webm',
220
221 # Apple HTTP Live Streaming
222 '92': 'mp4',
223 '93': 'mp4',
224 '94': 'mp4',
225 '95': 'mp4',
226 '96': 'mp4',
227 '132': 'mp4',
228 '151': 'mp4',
229
230 # Dash mp4
231 '133': 'mp4',
232 '134': 'mp4',
233 '135': 'mp4',
234 '136': 'mp4',
235 '137': 'mp4',
236 '138': 'mp4',
237 '139': 'mp4',
238 '140': 'mp4',
239 '141': 'mp4',
240 '160': 'mp4',
241
242 # Dash webm
243 '171': 'webm',
244 '172': 'webm',
245 '242': 'webm',
246 '243': 'webm',
247 '244': 'webm',
248 '245': 'webm',
249 '246': 'webm',
250 '247': 'webm',
251 '248': 'webm',
252 }
253 _video_dimensions = {
254 '5': '240x400',
255 '6': '???',
256 '13': '???',
257 '17': '144x176',
258 '18': '360x640',
259 '22': '720x1280',
260 '34': '360x640',
261 '35': '480x854',
262 '36': '240x320',
263 '37': '1080x1920',
264 '38': '3072x4096',
265 '43': '360x640',
266 '44': '480x854',
267 '45': '720x1280',
268 '46': '1080x1920',
269 '82': '360p',
270 '83': '480p',
271 '84': '720p',
272 '85': '1080p',
273 '92': '240p',
274 '93': '360p',
275 '94': '480p',
276 '95': '720p',
277 '96': '1080p',
278 '100': '360p',
279 '101': '480p',
280 '102': '720p',
281 '132': '240p',
282 '151': '72p',
283 '133': '240p',
284 '134': '360p',
285 '135': '480p',
286 '136': '720p',
287 '137': '1080p',
288 '138': '>1080p',
289 '139': '48k',
290 '140': '128k',
291 '141': '256k',
292 '160': '192p',
293 '171': '128k',
294 '172': '256k',
295 '242': '240p',
296 '243': '360p',
297 '244': '480p',
298 '245': '480p',
299 '246': '480p',
300 '247': '720p',
301 '248': '1080p',
302 }
303 _special_itags = {
304 '82': '3D',
305 '83': '3D',
306 '84': '3D',
307 '85': '3D',
308 '100': '3D',
309 '101': '3D',
310 '102': '3D',
311 '133': 'DASH Video',
312 '134': 'DASH Video',
313 '135': 'DASH Video',
314 '136': 'DASH Video',
315 '137': 'DASH Video',
316 '138': 'DASH Video',
317 '139': 'DASH Audio',
318 '140': 'DASH Audio',
319 '141': 'DASH Audio',
320 '160': 'DASH Video',
321 '171': 'DASH Audio',
322 '172': 'DASH Audio',
323 '242': 'DASH Video',
324 '243': 'DASH Video',
325 '244': 'DASH Video',
326 '245': 'DASH Video',
327 '246': 'DASH Video',
328 '247': 'DASH Video',
329 '248': 'DASH Video',
330 }
331
332 IE_NAME = u'youtube'
333 _TESTS = [
334 {
335 u"url": u"http://www.youtube.com/watch?v=BaW_jenozKc",
336 u"file": u"BaW_jenozKc.mp4",
337 u"info_dict": {
338 u"title": u"youtube-dl test video \"'/\\ä↭𝕐",
339 u"uploader": u"Philipp Hagemeister",
340 u"uploader_id": u"phihag",
341 u"upload_date": u"20121002",
342 u"description": u"test chars: \"'/\\ä↭𝕐\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de ."
343 }
344 },
345 {
346 u"url": u"http://www.youtube.com/watch?v=1ltcDfZMA3U",
347 u"file": u"1ltcDfZMA3U.flv",
348 u"note": u"Test VEVO video (#897)",
349 u"info_dict": {
350 u"upload_date": u"20070518",
351 u"title": u"Maps - It Will Find You",
352 u"description": u"Music video by Maps performing It Will Find You.",
353 u"uploader": u"MuteUSA",
354 u"uploader_id": u"MuteUSA"
355 }
356 },
357 {
358 u"url": u"http://www.youtube.com/watch?v=UxxajLWwzqY",
359 u"file": u"UxxajLWwzqY.mp4",
360 u"note": u"Test generic use_cipher_signature video (#897)",
361 u"info_dict": {
362 u"upload_date": u"20120506",
363 u"title": u"Icona Pop - I Love It (feat. Charli XCX) [OFFICIAL VIDEO]",
364 u"description": u"md5:3e2666e0a55044490499ea45fe9037b7",
365 u"uploader": u"Icona Pop",
366 u"uploader_id": u"IconaPop"
367 }
368 },
369 {
370 u"url": u"https://www.youtube.com/watch?v=07FYdnEawAQ",
371 u"file": u"07FYdnEawAQ.mp4",
372 u"note": u"Test VEVO video with age protection (#956)",
373 u"info_dict": {
374 u"upload_date": u"20130703",
375 u"title": u"Justin Timberlake - Tunnel Vision (Explicit)",
376 u"description": u"md5:64249768eec3bc4276236606ea996373",
377 u"uploader": u"justintimberlakeVEVO",
378 u"uploader_id": u"justintimberlakeVEVO"
379 }
380 },
381 {
382 u'url': u'https://www.youtube.com/watch?v=TGi3HqYrWHE',
383 u'file': u'TGi3HqYrWHE.mp4',
384 u'note': u'm3u8 video',
385 u'info_dict': {
386 u'title': u'Triathlon - Men - London 2012 Olympic Games',
387 u'description': u'- Men - TR02 - Triathlon - 07 August 2012 - London 2012 Olympic Games',
388 u'uploader': u'olympic',
389 u'upload_date': u'20120807',
390 u'uploader_id': u'olympic',
391 },
392 u'params': {
393 u'skip_download': True,
394 },
395 },
396 ]
397
398
399 @classmethod
400 def suitable(cls, url):
401 """Receives a URL and returns True if suitable for this IE."""
402 if YoutubePlaylistIE.suitable(url): return False
403 return re.match(cls._VALID_URL, url, re.VERBOSE) is not None
404
405 def __init__(self, *args, **kwargs):
406 super(YoutubeIE, self).__init__(*args, **kwargs)
407 self._player_cache = {}
408
409 def report_video_webpage_download(self, video_id):
410 """Report attempt to download video webpage."""
411 self.to_screen(u'%s: Downloading video webpage' % video_id)
412
413 def report_video_info_webpage_download(self, video_id):
414 """Report attempt to download video info webpage."""
415 self.to_screen(u'%s: Downloading video info webpage' % video_id)
416
417 def report_information_extraction(self, video_id):
418 """Report attempt to extract video information."""
419 self.to_screen(u'%s: Extracting video information' % video_id)
420
421 def report_unavailable_format(self, video_id, format):
422 """Report extracted video URL."""
423 self.to_screen(u'%s: Format %s not available' % (video_id, format))
424
425 def report_rtmp_download(self):
426 """Indicate the download will use the RTMP protocol."""
427 self.to_screen(u'RTMP download detected')
428
429 def _extract_signature_function(self, video_id, player_url, slen):
430 id_m = re.match(r'.*-(?P<id>[a-zA-Z0-9_-]+)\.(?P<ext>[a-z]+)$',
431 player_url)
432 player_type = id_m.group('ext')
433 player_id = id_m.group('id')
434
435 # Read from filesystem cache
436 func_id = '%s_%s_%d' % (player_type, player_id, slen)
437 assert os.path.basename(func_id) == func_id
438 cache_dir = self._downloader.params.get('cachedir',
439 u'~/.youtube-dl/cache')
440
441 cache_enabled = cache_dir != u'NONE'
442 if cache_enabled:
443 cache_fn = os.path.join(os.path.expanduser(cache_dir),
444 u'youtube-sigfuncs',
445 func_id + '.json')
446 try:
447 with io.open(cache_fn, 'r', encoding='utf-8') as cachef:
448 cache_spec = json.load(cachef)
449 return lambda s: u''.join(s[i] for i in cache_spec)
450 except IOError:
451 pass # No cache available
452
453 if player_type == 'js':
454 code = self._download_webpage(
455 player_url, video_id,
456 note=u'Downloading %s player %s' % (player_type, player_id),
457 errnote=u'Download of %s failed' % player_url)
458 res = self._parse_sig_js(code)
459 elif player_type == 'swf':
460 urlh = self._request_webpage(
461 player_url, video_id,
462 note=u'Downloading %s player %s' % (player_type, player_id),
463 errnote=u'Download of %s failed' % player_url)
464 code = urlh.read()
465 res = self._parse_sig_swf(code)
466 else:
467 assert False, 'Invalid player type %r' % player_type
468
469 if cache_enabled:
470 try:
471 cache_res = res(map(compat_chr, range(slen)))
472 cache_spec = [ord(c) for c in cache_res]
473 try:
474 os.makedirs(os.path.dirname(cache_fn))
475 except OSError as ose:
476 if ose.errno != errno.EEXIST:
477 raise
478 write_json_file(cache_spec, cache_fn)
479 except Exception:
480 tb = traceback.format_exc()
481 self._downloader.report_warning(
482 u'Writing cache to %r failed: %s' % (cache_fn, tb))
483
484 return res
485
486 def _print_sig_code(self, func, slen):
487 def gen_sig_code(idxs):
488 def _genslice(start, end, step):
489 starts = u'' if start == 0 else str(start)
490 ends = u':%d' % (end+step)
491 steps = u'' if step == 1 else (':%d' % step)
492 return u's[%s%s%s]' % (starts, ends, steps)
493
494 step = None
495 start = '(Never used)' # Quelch pyflakes warnings - start will be
496 # set as soon as step is set
497 for i, prev in zip(idxs[1:], idxs[:-1]):
498 if step is not None:
499 if i - prev == step:
500 continue
501 yield _genslice(start, prev, step)
502 step = None
503 continue
504 if i - prev in [-1, 1]:
505 step = i - prev
506 start = prev
507 continue
508 else:
509 yield u's[%d]' % prev
510 if step is None:
511 yield u's[%d]' % i
512 else:
513 yield _genslice(start, i, step)
514
515 cache_res = func(map(compat_chr, range(slen)))
516 cache_spec = [ord(c) for c in cache_res]
517 expr_code = u' + '.join(gen_sig_code(cache_spec))
518 code = u'if len(s) == %d:\n return %s\n' % (slen, expr_code)
519 self.to_screen(u'Extracted signature function:\n' + code)
520
521 def _parse_sig_js(self, jscode):
522 funcname = self._search_regex(
523 r'signature=([a-zA-Z]+)', jscode,
524 u'Initial JS player signature function name')
525
526 functions = {}
527
528 def argidx(varname):
529 return string.lowercase.index(varname)
530
531 def interpret_statement(stmt, local_vars, allow_recursion=20):
532 if allow_recursion < 0:
533 raise ExtractorError(u'Recursion limit reached')
534
535 if stmt.startswith(u'var '):
536 stmt = stmt[len(u'var '):]
537 ass_m = re.match(r'^(?P<out>[a-z]+)(?:\[(?P<index>[^\]]+)\])?' +
538 r'=(?P<expr>.*)$', stmt)
539 if ass_m:
540 if ass_m.groupdict().get('index'):
541 def assign(val):
542 lvar = local_vars[ass_m.group('out')]
543 idx = interpret_expression(ass_m.group('index'),
544 local_vars, allow_recursion)
545 assert isinstance(idx, int)
546 lvar[idx] = val
547 return val
548 expr = ass_m.group('expr')
549 else:
550 def assign(val):
551 local_vars[ass_m.group('out')] = val
552 return val
553 expr = ass_m.group('expr')
554 elif stmt.startswith(u'return '):
555 assign = lambda v: v
556 expr = stmt[len(u'return '):]
557 else:
558 raise ExtractorError(
559 u'Cannot determine left side of statement in %r' % stmt)
560
561 v = interpret_expression(expr, local_vars, allow_recursion)
562 return assign(v)
563
564 def interpret_expression(expr, local_vars, allow_recursion):
565 if expr.isdigit():
566 return int(expr)
567
568 if expr.isalpha():
569 return local_vars[expr]
570
571 m = re.match(r'^(?P<in>[a-z]+)\.(?P<member>.*)$', expr)
572 if m:
573 member = m.group('member')
574 val = local_vars[m.group('in')]
575 if member == 'split("")':
576 return list(val)
577 if member == 'join("")':
578 return u''.join(val)
579 if member == 'length':
580 return len(val)
581 if member == 'reverse()':
582 return val[::-1]
583 slice_m = re.match(r'slice\((?P<idx>.*)\)', member)
584 if slice_m:
585 idx = interpret_expression(
586 slice_m.group('idx'), local_vars, allow_recursion-1)
587 return val[idx:]
588
589 m = re.match(
590 r'^(?P<in>[a-z]+)\[(?P<idx>.+)\]$', expr)
591 if m:
592 val = local_vars[m.group('in')]
593 idx = interpret_expression(m.group('idx'), local_vars,
594 allow_recursion-1)
595 return val[idx]
596
597 m = re.match(r'^(?P<a>.+?)(?P<op>[%])(?P<b>.+?)$', expr)
598 if m:
599 a = interpret_expression(m.group('a'),
600 local_vars, allow_recursion)
601 b = interpret_expression(m.group('b'),
602 local_vars, allow_recursion)
603 return a % b
604
605 m = re.match(
606 r'^(?P<func>[a-zA-Z]+)\((?P<args>[a-z0-9,]+)\)$', expr)
607 if m:
608 fname = m.group('func')
609 if fname not in functions:
610 functions[fname] = extract_function(fname)
611 argvals = [int(v) if v.isdigit() else local_vars[v]
612 for v in m.group('args').split(',')]
613 return functions[fname](argvals)
614 raise ExtractorError(u'Unsupported JS expression %r' % expr)
615
616 def extract_function(funcname):
617 func_m = re.search(
618 r'function ' + re.escape(funcname) +
619 r'\((?P<args>[a-z,]+)\){(?P<code>[^}]+)}',
620 jscode)
621 argnames = func_m.group('args').split(',')
622
623 def resf(args):
624 local_vars = dict(zip(argnames, args))
625 for stmt in func_m.group('code').split(';'):
626 res = interpret_statement(stmt, local_vars)
627 return res
628 return resf
629
630 initial_function = extract_function(funcname)
631 return lambda s: initial_function([s])
632
633 def _parse_sig_swf(self, file_contents):
634 if file_contents[1:3] != b'WS':
635 raise ExtractorError(
636 u'Not an SWF file; header is %r' % file_contents[:3])
637 if file_contents[:1] == b'C':
638 content = zlib.decompress(file_contents[8:])
639 else:
640 raise NotImplementedError(u'Unsupported compression format %r' %
641 file_contents[:1])
642
643 def extract_tags(content):
644 pos = 0
645 while pos < len(content):
646 header16 = struct.unpack('<H', content[pos:pos+2])[0]
647 pos += 2
648 tag_code = header16 >> 6
649 tag_len = header16 & 0x3f
650 if tag_len == 0x3f:
651 tag_len = struct.unpack('<I', content[pos:pos+4])[0]
652 pos += 4
653 assert pos+tag_len <= len(content)
654 yield (tag_code, content[pos:pos+tag_len])
655 pos += tag_len
656
657 code_tag = next(tag
658 for tag_code, tag in extract_tags(content)
659 if tag_code == 82)
660 p = code_tag.index(b'\0', 4) + 1
661 code_reader = io.BytesIO(code_tag[p:])
662
663 # Parse ABC (AVM2 ByteCode)
664 def read_int(reader=None):
665 if reader is None:
666 reader = code_reader
667 res = 0
668 shift = 0
669 for _ in range(5):
670 buf = reader.read(1)
671 assert len(buf) == 1
672 b = struct.unpack('<B', buf)[0]
673 res = res | ((b & 0x7f) << shift)
674 if b & 0x80 == 0:
675 break
676 shift += 7
677 return res
678
679 def u30(reader=None):
680 res = read_int(reader)
681 assert res & 0xf0000000 == 0
682 return res
683 u32 = read_int
684
685 def s32(reader=None):
686 v = read_int(reader)
687 if v & 0x80000000 != 0:
688 v = - ((v ^ 0xffffffff) + 1)
689 return v
690
691 def read_string(reader=None):
692 if reader is None:
693 reader = code_reader
694 slen = u30(reader)
695 resb = reader.read(slen)
696 assert len(resb) == slen
697 return resb.decode('utf-8')
698
699 def read_bytes(count, reader=None):
700 if reader is None:
701 reader = code_reader
702 resb = reader.read(count)
703 assert len(resb) == count
704 return resb
705
706 def read_byte(reader=None):
707 resb = read_bytes(1, reader=reader)
708 res = struct.unpack('<B', resb)[0]
709 return res
710
711 # minor_version + major_version
712 read_bytes(2 + 2)
713
714 # Constant pool
715 int_count = u30()
716 for _c in range(1, int_count):
717 s32()
718 uint_count = u30()
719 for _c in range(1, uint_count):
720 u32()
721 double_count = u30()
722 read_bytes((double_count-1) * 8)
723 string_count = u30()
724 constant_strings = [u'']
725 for _c in range(1, string_count):
726 s = read_string()
727 constant_strings.append(s)
728 namespace_count = u30()
729 for _c in range(1, namespace_count):
730 read_bytes(1) # kind
731 u30() # name
732 ns_set_count = u30()
733 for _c in range(1, ns_set_count):
734 count = u30()
735 for _c2 in range(count):
736 u30()
737 multiname_count = u30()
738 MULTINAME_SIZES = {
739 0x07: 2, # QName
740 0x0d: 2, # QNameA
741 0x0f: 1, # RTQName
742 0x10: 1, # RTQNameA
743 0x11: 0, # RTQNameL
744 0x12: 0, # RTQNameLA
745 0x09: 2, # Multiname
746 0x0e: 2, # MultinameA
747 0x1b: 1, # MultinameL
748 0x1c: 1, # MultinameLA
749 }
750 multinames = [u'']
751 for _c in range(1, multiname_count):
752 kind = u30()
753 assert kind in MULTINAME_SIZES, u'Invalid multiname kind %r' % kind
754 if kind == 0x07:
755 u30() # namespace_idx
756 name_idx = u30()
757 multinames.append(constant_strings[name_idx])
758 else:
759 multinames.append('[MULTINAME kind: %d]' % kind)
760 for _c2 in range(MULTINAME_SIZES[kind]):
761 u30()
762
763 # Methods
764 method_count = u30()
765 MethodInfo = collections.namedtuple(
766 'MethodInfo',
767 ['NEED_ARGUMENTS', 'NEED_REST'])
768 method_infos = []
769 for method_id in range(method_count):
770 param_count = u30()
771 u30() # return type
772 for _ in range(param_count):
773 u30() # param type
774 u30() # name index (always 0 for youtube)
775 flags = read_byte()
776 if flags & 0x08 != 0:
777 # Options present
778 option_count = u30()
779 for c in range(option_count):
780 u30() # val
781 read_bytes(1) # kind
782 if flags & 0x80 != 0:
783 # Param names present
784 for _ in range(param_count):
785 u30() # param name
786 mi = MethodInfo(flags & 0x01 != 0, flags & 0x04 != 0)
787 method_infos.append(mi)
788
789 # Metadata
790 metadata_count = u30()
791 for _c in range(metadata_count):
792 u30() # name
793 item_count = u30()
794 for _c2 in range(item_count):
795 u30() # key
796 u30() # value
797
798 def parse_traits_info():
799 trait_name_idx = u30()
800 kind_full = read_byte()
801 kind = kind_full & 0x0f
802 attrs = kind_full >> 4
803 methods = {}
804 if kind in [0x00, 0x06]: # Slot or Const
805 u30() # Slot id
806 u30() # type_name_idx
807 vindex = u30()
808 if vindex != 0:
809 read_byte() # vkind
810 elif kind in [0x01, 0x02, 0x03]: # Method / Getter / Setter
811 u30() # disp_id
812 method_idx = u30()
813 methods[multinames[trait_name_idx]] = method_idx
814 elif kind == 0x04: # Class
815 u30() # slot_id
816 u30() # classi
817 elif kind == 0x05: # Function
818 u30() # slot_id
819 function_idx = u30()
820 methods[function_idx] = multinames[trait_name_idx]
821 else:
822 raise ExtractorError(u'Unsupported trait kind %d' % kind)
823
824 if attrs & 0x4 != 0: # Metadata present
825 metadata_count = u30()
826 for _c3 in range(metadata_count):
827 u30() # metadata index
828
829 return methods
830
831 # Classes
832 TARGET_CLASSNAME = u'SignatureDecipher'
833 searched_idx = multinames.index(TARGET_CLASSNAME)
834 searched_class_id = None
835 class_count = u30()
836 for class_id in range(class_count):
837 name_idx = u30()
838 if name_idx == searched_idx:
839 # We found the class we're looking for!
840 searched_class_id = class_id
841 u30() # super_name idx
842 flags = read_byte()
843 if flags & 0x08 != 0: # Protected namespace is present
844 u30() # protected_ns_idx
845 intrf_count = u30()
846 for _c2 in range(intrf_count):
847 u30()
848 u30() # iinit
849 trait_count = u30()
850 for _c2 in range(trait_count):
851 parse_traits_info()
852
853 if searched_class_id is None:
854 raise ExtractorError(u'Target class %r not found' %
855 TARGET_CLASSNAME)
856
857 method_names = {}
858 method_idxs = {}
859 for class_id in range(class_count):
860 u30() # cinit
861 trait_count = u30()
862 for _c2 in range(trait_count):
863 trait_methods = parse_traits_info()
864 if class_id == searched_class_id:
865 method_names.update(trait_methods.items())
866 method_idxs.update(dict(
867 (idx, name)
868 for name, idx in trait_methods.items()))
869
870 # Scripts
871 script_count = u30()
872 for _c in range(script_count):
873 u30() # init
874 trait_count = u30()
875 for _c2 in range(trait_count):
876 parse_traits_info()
877
878 # Method bodies
879 method_body_count = u30()
880 Method = collections.namedtuple('Method', ['code', 'local_count'])
881 methods = {}
882 for _c in range(method_body_count):
883 method_idx = u30()
884 u30() # max_stack
885 local_count = u30()
886 u30() # init_scope_depth
887 u30() # max_scope_depth
888 code_length = u30()
889 code = read_bytes(code_length)
890 if method_idx in method_idxs:
891 m = Method(code, local_count)
892 methods[method_idxs[method_idx]] = m
893 exception_count = u30()
894 for _c2 in range(exception_count):
895 u30() # from
896 u30() # to
897 u30() # target
898 u30() # exc_type
899 u30() # var_name
900 trait_count = u30()
901 for _c2 in range(trait_count):
902 parse_traits_info()
903
904 assert p + code_reader.tell() == len(code_tag)
905 assert len(methods) == len(method_idxs)
906
907 method_pyfunctions = {}
908
909 def extract_function(func_name):
910 if func_name in method_pyfunctions:
911 return method_pyfunctions[func_name]
912 if func_name not in methods:
913 raise ExtractorError(u'Cannot find function %r' % func_name)
914 m = methods[func_name]
915
916 def resfunc(args):
917 registers = ['(this)'] + list(args) + [None] * m.local_count
918 stack = []
919 coder = io.BytesIO(m.code)
920 while True:
921 opcode = struct.unpack('!B', coder.read(1))[0]
922 if opcode == 36: # pushbyte
923 v = struct.unpack('!B', coder.read(1))[0]
924 stack.append(v)
925 elif opcode == 44: # pushstring
926 idx = u30(coder)
927 stack.append(constant_strings[idx])
928 elif opcode == 48: # pushscope
929 # We don't implement the scope register, so we'll just
930 # ignore the popped value
931 stack.pop()
932 elif opcode == 70: # callproperty
933 index = u30(coder)
934 mname = multinames[index]
935 arg_count = u30(coder)
936 args = list(reversed(
937 [stack.pop() for _ in range(arg_count)]))
938 obj = stack.pop()
939 if mname == u'split':
940 assert len(args) == 1
941 assert isinstance(args[0], compat_str)
942 assert isinstance(obj, compat_str)
943 if args[0] == u'':
944 res = list(obj)
945 else:
946 res = obj.split(args[0])
947 stack.append(res)
948 elif mname == u'slice':
949 assert len(args) == 1
950 assert isinstance(args[0], int)
951 assert isinstance(obj, list)
952 res = obj[args[0]:]
953 stack.append(res)
954 elif mname == u'join':
955 assert len(args) == 1
956 assert isinstance(args[0], compat_str)
957 assert isinstance(obj, list)
958 res = args[0].join(obj)
959 stack.append(res)
960 elif mname in method_pyfunctions:
961 stack.append(method_pyfunctions[mname](args))
962 else:
963 raise NotImplementedError(
964 u'Unsupported property %r on %r'
965 % (mname, obj))
966 elif opcode == 72: # returnvalue
967 res = stack.pop()
968 return res
969 elif opcode == 79: # callpropvoid
970 index = u30(coder)
971 mname = multinames[index]
972 arg_count = u30(coder)
973 args = list(reversed(
974 [stack.pop() for _ in range(arg_count)]))
975 obj = stack.pop()
976 if mname == u'reverse':
977 assert isinstance(obj, list)
978 obj.reverse()
979 else:
980 raise NotImplementedError(
981 u'Unsupported (void) property %r on %r'
982 % (mname, obj))
983 elif opcode == 93: # findpropstrict
984 index = u30(coder)
985 mname = multinames[index]
986 res = extract_function(mname)
987 stack.append(res)
988 elif opcode == 97: # setproperty
989 index = u30(coder)
990 value = stack.pop()
991 idx = stack.pop()
992 obj = stack.pop()
993 assert isinstance(obj, list)
994 assert isinstance(idx, int)
995 obj[idx] = value
996 elif opcode == 98: # getlocal
997 index = u30(coder)
998 stack.append(registers[index])
999 elif opcode == 99: # setlocal
1000 index = u30(coder)
1001 value = stack.pop()
1002 registers[index] = value
1003 elif opcode == 102: # getproperty
1004 index = u30(coder)
1005 pname = multinames[index]
1006 if pname == u'length':
1007 obj = stack.pop()
1008 assert isinstance(obj, list)
1009 stack.append(len(obj))
1010 else: # Assume attribute access
1011 idx = stack.pop()
1012 assert isinstance(idx, int)
1013 obj = stack.pop()
1014 assert isinstance(obj, list)
1015 stack.append(obj[idx])
1016 elif opcode == 128: # coerce
1017 u30(coder)
1018 elif opcode == 133: # coerce_s
1019 assert isinstance(stack[-1], (type(None), compat_str))
1020 elif opcode == 164: # modulo
1021 value2 = stack.pop()
1022 value1 = stack.pop()
1023 res = value1 % value2
1024 stack.append(res)
1025 elif opcode == 208: # getlocal_0
1026 stack.append(registers[0])
1027 elif opcode == 209: # getlocal_1
1028 stack.append(registers[1])
1029 elif opcode == 210: # getlocal_2
1030 stack.append(registers[2])
1031 elif opcode == 211: # getlocal_3
1032 stack.append(registers[3])
1033 elif opcode == 214: # setlocal_2
1034 registers[2] = stack.pop()
1035 elif opcode == 215: # setlocal_3
1036 registers[3] = stack.pop()
1037 else:
1038 raise NotImplementedError(
1039 u'Unsupported opcode %d' % opcode)
1040
1041 method_pyfunctions[func_name] = resfunc
1042 return resfunc
1043
1044 initial_function = extract_function(u'decipher')
1045 return lambda s: initial_function([s])
1046
1047 def _decrypt_signature(self, s, video_id, player_url, age_gate=False):
1048 """Turn the encrypted s field into a working signature"""
1049
1050 if player_url is not None:
1051 try:
1052 if player_url not in self._player_cache:
1053 func = self._extract_signature_function(
1054 video_id, player_url, len(s)
1055 )
1056 self._player_cache[player_url] = func
1057 func = self._player_cache[player_url]
1058 if self._downloader.params.get('youtube_print_sig_code'):
1059 self._print_sig_code(func, len(s))
1060 return func(s)
1061 except Exception:
1062 tb = traceback.format_exc()
1063 self._downloader.report_warning(
1064 u'Automatic signature extraction failed: ' + tb)
1065
1066 self._downloader.report_warning(
1067 u'Warning: Falling back to static signature algorithm')
1068 return self._static_decrypt_signature(
1069 s, video_id, player_url, age_gate)
1070
1071 def _static_decrypt_signature(self, s, video_id, player_url, age_gate):
1072 if age_gate:
1073 # The videos with age protection use another player, so the
1074 # algorithms can be different.
1075 if len(s) == 86:
1076 return s[2:63] + s[82] + s[64:82] + s[63]
1077
1078 if len(s) == 93:
1079 return s[86:29:-1] + s[88] + s[28:5:-1]
1080 elif len(s) == 92:
1081 return s[25] + s[3:25] + s[0] + s[26:42] + s[79] + s[43:79] + s[91] + s[80:83]
1082 elif len(s) == 91:
1083 return s[84:27:-1] + s[86] + s[26:5:-1]
1084 elif len(s) == 90:
1085 return s[25] + s[3:25] + s[2] + s[26:40] + s[77] + s[41:77] + s[89] + s[78:81]
1086 elif len(s) == 89:
1087 return s[84:78:-1] + s[87] + s[77:60:-1] + s[0] + s[59:3:-1]
1088 elif len(s) == 88:
1089 return s[7:28] + s[87] + s[29:45] + s[55] + s[46:55] + s[2] + s[56:87] + s[28]
1090 elif len(s) == 87:
1091 return s[6:27] + s[4] + s[28:39] + s[27] + s[40:59] + s[2] + s[60:]
1092 elif len(s) == 86:
1093 return s[5:34] + s[0] + s[35:38] + s[3] + s[39:45] + s[38] + s[46:53] + s[73] + s[54:73] + s[85] + s[74:85] + s[53]
1094 elif len(s) == 85:
1095 return s[3:11] + s[0] + s[12:55] + s[84] + s[56:84]
1096 elif len(s) == 84:
1097 return s[81:36:-1] + s[0] + s[35:2:-1]
1098 elif len(s) == 83:
1099 return s[81:64:-1] + s[82] + s[63:52:-1] + s[45] + s[51:45:-1] + s[1] + s[44:1:-1] + s[0]
1100 elif len(s) == 82:
1101 return s[80:73:-1] + s[81] + s[72:54:-1] + s[2] + s[53:43:-1] + s[0] + s[42:2:-1] + s[43] + s[1] + s[54]
1102 elif len(s) == 81:
1103 return s[56] + s[79:56:-1] + s[41] + s[55:41:-1] + s[80] + s[40:34:-1] + s[0] + s[33:29:-1] + s[34] + s[28:9:-1] + s[29] + s[8:0:-1] + s[9]
1104 elif len(s) == 80:
1105 return s[1:19] + s[0] + s[20:68] + s[19] + s[69:80]
1106 elif len(s) == 79:
1107 return s[54] + s[77:54:-1] + s[39] + s[53:39:-1] + s[78] + s[38:34:-1] + s[0] + s[33:29:-1] + s[34] + s[28:9:-1] + s[29] + s[8:0:-1] + s[9]
1108
1109 else:
1110 raise ExtractorError(u'Unable to decrypt signature, key length %d not supported; retrying might work' % (len(s)))
1111
1112 def _get_available_subtitles(self, video_id):
1113 try:
1114 sub_list = self._download_webpage(
1115 'http://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id,
1116 video_id, note=False)
1117 except ExtractorError as err:
1118 self._downloader.report_warning(u'unable to download video subtitles: %s' % compat_str(err))
1119 return {}
1120 lang_list = re.findall(r'name="([^"]*)"[^>]+lang_code="([\w\-]+)"', sub_list)
1121
1122 sub_lang_list = {}
1123 for l in lang_list:
1124 lang = l[1]
1125 params = compat_urllib_parse.urlencode({
1126 'lang': lang,
1127 'v': video_id,
1128 'fmt': self._downloader.params.get('subtitlesformat'),
1129 })
1130 url = u'http://www.youtube.com/api/timedtext?' + params
1131 sub_lang_list[lang] = url
1132 if not sub_lang_list:
1133 self._downloader.report_warning(u'video doesn\'t have subtitles')
1134 return {}
1135 return sub_lang_list
1136
1137 def _get_available_automatic_caption(self, video_id, webpage):
1138 """We need the webpage for getting the captions url, pass it as an
1139 argument to speed up the process."""
1140 sub_format = self._downloader.params.get('subtitlesformat')
1141 self.to_screen(u'%s: Looking for automatic captions' % video_id)
1142 mobj = re.search(r';ytplayer.config = ({.*?});', webpage)
1143 err_msg = u'Couldn\'t find automatic captions for %s' % video_id
1144 if mobj is None:
1145 self._downloader.report_warning(err_msg)
1146 return {}
1147 player_config = json.loads(mobj.group(1))
1148 try:
1149 args = player_config[u'args']
1150 caption_url = args[u'ttsurl']
1151 timestamp = args[u'timestamp']
1152 # We get the available subtitles
1153 list_params = compat_urllib_parse.urlencode({
1154 'type': 'list',
1155 'tlangs': 1,
1156 'asrs': 1,
1157 })
1158 list_url = caption_url + '&' + list_params
1159 list_page = self._download_webpage(list_url, video_id)
1160 caption_list = xml.etree.ElementTree.fromstring(list_page.encode('utf-8'))
1161 original_lang_node = caption_list.find('track')
1162 if original_lang_node.attrib.get('kind') != 'asr' :
1163 self._downloader.report_warning(u'Video doesn\'t have automatic captions')
1164 return {}
1165 original_lang = original_lang_node.attrib['lang_code']
1166
1167 sub_lang_list = {}
1168 for lang_node in caption_list.findall('target'):
1169 sub_lang = lang_node.attrib['lang_code']
1170 params = compat_urllib_parse.urlencode({
1171 'lang': original_lang,
1172 'tlang': sub_lang,
1173 'fmt': sub_format,
1174 'ts': timestamp,
1175 'kind': 'asr',
1176 })
1177 sub_lang_list[sub_lang] = caption_url + '&' + params
1178 return sub_lang_list
1179 # An extractor error can be raise by the download process if there are
1180 # no automatic captions but there are subtitles
1181 except (KeyError, ExtractorError):
1182 self._downloader.report_warning(err_msg)
1183 return {}
1184
1185 def _print_formats(self, formats):
1186 print('Available formats:')
1187 for x in formats:
1188 print('%s\t:\t%s\t[%s]%s' %(x, self._video_extensions.get(x, 'flv'),
1189 self._video_dimensions.get(x, '???'),
1190 ' ('+self._special_itags[x]+')' if x in self._special_itags else ''))
1191
1192 def _extract_id(self, url):
1193 mobj = re.match(self._VALID_URL, url, re.VERBOSE)
1194 if mobj is None:
1195 raise ExtractorError(u'Invalid URL: %s' % url)
1196 video_id = mobj.group(2)
1197 return video_id
1198
1199 def _get_video_url_list(self, url_map):
1200 """
1201 Transform a dictionary in the format {itag:url} to a list of (itag, url)
1202 with the requested formats.
1203 """
1204 req_format = self._downloader.params.get('format', None)
1205 format_limit = self._downloader.params.get('format_limit', None)
1206 available_formats = self._available_formats_prefer_free if self._downloader.params.get('prefer_free_formats', False) else self._available_formats
1207 if format_limit is not None and format_limit in available_formats:
1208 format_list = available_formats[available_formats.index(format_limit):]
1209 else:
1210 format_list = available_formats
1211 existing_formats = [x for x in format_list if x in url_map]
1212 if len(existing_formats) == 0:
1213 raise ExtractorError(u'no known formats available for video')
1214 if self._downloader.params.get('listformats', None):
1215 self._print_formats(existing_formats)
1216 return
1217 if req_format is None or req_format == 'best':
1218 video_url_list = [(existing_formats[0], url_map[existing_formats[0]])] # Best quality
1219 elif req_format == 'worst':
1220 video_url_list = [(existing_formats[-1], url_map[existing_formats[-1]])] # worst quality
1221 elif req_format in ('-1', 'all'):
1222 video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats
1223 else:
1224 # Specific formats. We pick the first in a slash-delimeted sequence.
1225 # Format can be specified as itag or 'mp4' or 'flv' etc. We pick the highest quality
1226 # available in the specified format. For example,
1227 # if '1/2/3/4' is requested and '2' and '4' are available, we pick '2'.
1228 # if '1/mp4/3/4' is requested and '1' and '5' (is a mp4) are available, we pick '1'.
1229 # if '1/mp4/3/4' is requested and '4' and '5' (is a mp4) are available, we pick '5'.
1230 req_formats = req_format.split('/')
1231 video_url_list = None
1232 for rf in req_formats:
1233 if rf in url_map:
1234 video_url_list = [(rf, url_map[rf])]
1235 break
1236 if rf in self._video_formats_map:
1237 for srf in self._video_formats_map[rf]:
1238 if srf in url_map:
1239 video_url_list = [(srf, url_map[srf])]
1240 break
1241 else:
1242 continue
1243 break
1244 if video_url_list is None:
1245 raise ExtractorError(u'requested format not available')
1246 return video_url_list
1247
1248 def _extract_from_m3u8(self, manifest_url, video_id):
1249 url_map = {}
1250 def _get_urls(_manifest):
1251 lines = _manifest.split('\n')
1252 urls = filter(lambda l: l and not l.startswith('#'),
1253 lines)
1254 return urls
1255 manifest = self._download_webpage(manifest_url, video_id, u'Downloading formats manifest')
1256 formats_urls = _get_urls(manifest)
1257 for format_url in formats_urls:
1258 itag = self._search_regex(r'itag/(\d+?)/', format_url, 'itag')
1259 url_map[itag] = format_url
1260 return url_map
1261
1262 def _real_extract(self, url):
1263 if re.match(r'(?:https?://)?[^/]+/watch\?feature=[a-z_]+$', url):
1264 self._downloader.report_warning(u'Did you forget to quote the URL? Remember that & is a meta-character in most shells, so you want to put the URL in quotes, like youtube-dl \'http://www.youtube.com/watch?feature=foo&v=BaW_jenozKc\' (or simply youtube-dl BaW_jenozKc ).')
1265
1266 # Extract original video URL from URL with redirection, like age verification, using next_url parameter
1267 mobj = re.search(self._NEXT_URL_RE, url)
1268 if mobj:
1269 url = 'https://www.youtube.com/' + compat_urllib_parse.unquote(mobj.group(1)).lstrip('/')
1270 video_id = self._extract_id(url)
1271
1272 # Get video webpage
1273 self.report_video_webpage_download(video_id)
1274 url = 'https://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1' % video_id
1275 request = compat_urllib_request.Request(url)
1276 try:
1277 video_webpage_bytes = compat_urllib_request.urlopen(request).read()
1278 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1279 raise ExtractorError(u'Unable to download video webpage: %s' % compat_str(err))
1280
1281 video_webpage = video_webpage_bytes.decode('utf-8', 'ignore')
1282
1283 # Attempt to extract SWF player URL
1284 mobj = re.search(r'swfConfig.*?"(https?:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage)
1285 if mobj is not None:
1286 player_url = re.sub(r'\\(.)', r'\1', mobj.group(1))
1287 else:
1288 player_url = None
1289
1290 # Get video info
1291 self.report_video_info_webpage_download(video_id)
1292 if re.search(r'player-age-gate-content">', video_webpage) is not None:
1293 self.report_age_confirmation()
1294 age_gate = True
1295 # We simulate the access to the video from www.youtube.com/v/{video_id}
1296 # this can be viewed without login into Youtube
1297 data = compat_urllib_parse.urlencode({'video_id': video_id,
1298 'el': 'embedded',
1299 'gl': 'US',
1300 'hl': 'en',
1301 'eurl': 'https://youtube.googleapis.com/v/' + video_id,
1302 'asv': 3,
1303 'sts':'1588',
1304 })
1305 video_info_url = 'https://www.youtube.com/get_video_info?' + data
1306 video_info_webpage = self._download_webpage(video_info_url, video_id,
1307 note=False,
1308 errnote='unable to download video info webpage')
1309 video_info = compat_parse_qs(video_info_webpage)
1310 else:
1311 age_gate = False
1312 for el_type in ['&el=embedded', '&el=detailpage', '&el=vevo', '']:
1313 video_info_url = ('https://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
1314 % (video_id, el_type))
1315 video_info_webpage = self._download_webpage(video_info_url, video_id,
1316 note=False,
1317 errnote='unable to download video info webpage')
1318 video_info = compat_parse_qs(video_info_webpage)
1319 if 'token' in video_info:
1320 break
1321 if 'token' not in video_info:
1322 if 'reason' in video_info:
1323 raise ExtractorError(u'YouTube said: %s' % video_info['reason'][0], expected=True)
1324 else:
1325 raise ExtractorError(u'"token" parameter not in video info for unknown reason')
1326
1327 # Check for "rental" videos
1328 if 'ypc_video_rental_bar_text' in video_info and 'author' not in video_info:
1329 raise ExtractorError(u'"rental" videos not supported')
1330
1331 # Start extracting information
1332 self.report_information_extraction(video_id)
1333
1334 # uploader
1335 if 'author' not in video_info:
1336 raise ExtractorError(u'Unable to extract uploader name')
1337 video_uploader = compat_urllib_parse.unquote_plus(video_info['author'][0])
1338
1339 # uploader_id
1340 video_uploader_id = None
1341 mobj = re.search(r'<link itemprop="url" href="http://www.youtube.com/(?:user|channel)/([^"]+)">', video_webpage)
1342 if mobj is not None:
1343 video_uploader_id = mobj.group(1)
1344 else:
1345 self._downloader.report_warning(u'unable to extract uploader nickname')
1346
1347 # title
1348 if 'title' not in video_info:
1349 raise ExtractorError(u'Unable to extract video title')
1350 video_title = compat_urllib_parse.unquote_plus(video_info['title'][0])
1351
1352 # thumbnail image
1353 # We try first to get a high quality image:
1354 m_thumb = re.search(r'<span itemprop="thumbnail".*?href="(.*?)">',
1355 video_webpage, re.DOTALL)
1356 if m_thumb is not None:
1357 video_thumbnail = m_thumb.group(1)
1358 elif 'thumbnail_url' not in video_info:
1359 self._downloader.report_warning(u'unable to extract video thumbnail')
1360 video_thumbnail = ''
1361 else: # don't panic if we can't find it
1362 video_thumbnail = compat_urllib_parse.unquote_plus(video_info['thumbnail_url'][0])
1363
1364 # upload date
1365 upload_date = None
1366 mobj = re.search(r'id="eow-date.*?>(.*?)</span>', video_webpage, re.DOTALL)
1367 if mobj is not None:
1368 upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split())
1369 upload_date = unified_strdate(upload_date)
1370
1371 # description
1372 video_description = get_element_by_id("eow-description", video_webpage)
1373 if video_description:
1374 video_description = clean_html(video_description)
1375 else:
1376 fd_mobj = re.search(r'<meta name="description" content="([^"]+)"', video_webpage)
1377 if fd_mobj:
1378 video_description = unescapeHTML(fd_mobj.group(1))
1379 else:
1380 video_description = u''
1381
1382 # subtitles
1383 video_subtitles = self.extract_subtitles(video_id, video_webpage)
1384
1385 if self._downloader.params.get('listsubtitles', False):
1386 self._list_available_subtitles(video_id, video_webpage)
1387 return
1388
1389 if 'length_seconds' not in video_info:
1390 self._downloader.report_warning(u'unable to extract video duration')
1391 video_duration = ''
1392 else:
1393 video_duration = compat_urllib_parse.unquote_plus(video_info['length_seconds'][0])
1394
1395 # Decide which formats to download
1396
1397 try:
1398 mobj = re.search(r';ytplayer.config = ({.*?});', video_webpage)
1399 if not mobj:
1400 raise ValueError('Could not find vevo ID')
1401 info = json.loads(mobj.group(1))
1402 args = info['args']
1403 # Easy way to know if the 's' value is in url_encoded_fmt_stream_map
1404 # this signatures are encrypted
1405 m_s = re.search(r'[&,]s=', args['url_encoded_fmt_stream_map'])
1406 if m_s is not None:
1407 self.to_screen(u'%s: Encrypted signatures detected.' % video_id)
1408 video_info['url_encoded_fmt_stream_map'] = [args['url_encoded_fmt_stream_map']]
1409 m_s = re.search(r'[&,]s=', args.get('adaptive_fmts', u''))
1410 if m_s is not None:
1411 if 'url_encoded_fmt_stream_map' in video_info:
1412 video_info['url_encoded_fmt_stream_map'][0] += ',' + args['adaptive_fmts']
1413 else:
1414 video_info['url_encoded_fmt_stream_map'] = [args['adaptive_fmts']]
1415 elif 'adaptive_fmts' in video_info:
1416 if 'url_encoded_fmt_stream_map' in video_info:
1417 video_info['url_encoded_fmt_stream_map'][0] += ',' + video_info['adaptive_fmts'][0]
1418 else:
1419 video_info['url_encoded_fmt_stream_map'] = video_info['adaptive_fmts']
1420 except ValueError:
1421 pass
1422
1423 if 'conn' in video_info and video_info['conn'][0].startswith('rtmp'):
1424 self.report_rtmp_download()
1425 video_url_list = [(None, video_info['conn'][0])]
1426 elif 'url_encoded_fmt_stream_map' in video_info and len(video_info['url_encoded_fmt_stream_map']) >= 1:
1427 if 'rtmpe%3Dyes' in video_info['url_encoded_fmt_stream_map'][0]:
1428 raise ExtractorError('rtmpe downloads are not supported, see https://github.com/rg3/youtube-dl/issues/343 for more information.', expected=True)
1429 url_map = {}
1430 for url_data_str in video_info['url_encoded_fmt_stream_map'][0].split(','):
1431 url_data = compat_parse_qs(url_data_str)
1432 if 'itag' in url_data and 'url' in url_data:
1433 url = url_data['url'][0]
1434 if 'sig' in url_data:
1435 url += '&signature=' + url_data['sig'][0]
1436 elif 's' in url_data:
1437 encrypted_sig = url_data['s'][0]
1438 if self._downloader.params.get('verbose'):
1439 if age_gate:
1440 if player_url is None:
1441 player_version = 'unknown'
1442 else:
1443 player_version = self._search_regex(
1444 r'-(.+)\.swf$', player_url,
1445 u'flash player', fatal=False)
1446 player_desc = 'flash player %s' % player_version
1447 else:
1448 player_version = self._search_regex(
1449 r'html5player-(.+?)\.js', video_webpage,
1450 'html5 player', fatal=False)
1451 player_desc = u'html5 player %s' % player_version
1452
1453 parts_sizes = u'.'.join(compat_str(len(part)) for part in encrypted_sig.split('.'))
1454 self.to_screen(u'encrypted signature length %d (%s), itag %s, %s' %
1455 (len(encrypted_sig), parts_sizes, url_data['itag'][0], player_desc))
1456
1457 if not age_gate:
1458 jsplayer_url_json = self._search_regex(
1459 r'"assets":.+?"js":\s*("[^"]+")',
1460 video_webpage, u'JS player URL')
1461 player_url = json.loads(jsplayer_url_json)
1462
1463 signature = self._decrypt_signature(
1464 encrypted_sig, video_id, player_url, age_gate)
1465 url += '&signature=' + signature
1466 if 'ratebypass' not in url:
1467 url += '&ratebypass=yes'
1468 url_map[url_data['itag'][0]] = url
1469 video_url_list = self._get_video_url_list(url_map)
1470 if not video_url_list:
1471 return
1472 elif video_info.get('hlsvp'):
1473 manifest_url = video_info['hlsvp'][0]
1474 url_map = self._extract_from_m3u8(manifest_url, video_id)
1475 video_url_list = self._get_video_url_list(url_map)
1476 if not video_url_list:
1477 return
1478
1479 else:
1480 raise ExtractorError(u'no conn or url_encoded_fmt_stream_map information found in video info')
1481
1482 results = []
1483 for format_param, video_real_url in video_url_list:
1484 # Extension
1485 video_extension = self._video_extensions.get(format_param, 'flv')
1486
1487 video_format = '{0} - {1}{2}'.format(format_param if format_param else video_extension,
1488 self._video_dimensions.get(format_param, '???'),
1489 ' ('+self._special_itags[format_param]+')' if format_param in self._special_itags else '')
1490
1491 results.append({
1492 'id': video_id,
1493 'url': video_real_url,
1494 'uploader': video_uploader,
1495 'uploader_id': video_uploader_id,
1496 'upload_date': upload_date,
1497 'title': video_title,
1498 'ext': video_extension,
1499 'format': video_format,
1500 'thumbnail': video_thumbnail,
1501 'description': video_description,
1502 'player_url': player_url,
1503 'subtitles': video_subtitles,
1504 'duration': video_duration
1505 })
1506 return results
1507
1508 class YoutubePlaylistIE(InfoExtractor):
1509 IE_DESC = u'YouTube.com playlists'
1510 _VALID_URL = r"""(?:
1511 (?:https?://)?
1512 (?:\w+\.)?
1513 youtube\.com/
1514 (?:
1515 (?:course|view_play_list|my_playlists|artist|playlist|watch)
1516 \? (?:.*?&)*? (?:p|a|list)=
1517 | p/
1518 )
1519 ((?:PL|EC|UU|FL)?[0-9A-Za-z-_]{10,})
1520 .*
1521 |
1522 ((?:PL|EC|UU|FL)[0-9A-Za-z-_]{10,})
1523 )"""
1524 _TEMPLATE_URL = 'https://gdata.youtube.com/feeds/api/playlists/%s?max-results=%i&start-index=%i&v=2&alt=json&safeSearch=none'
1525 _MAX_RESULTS = 50
1526 IE_NAME = u'youtube:playlist'
1527
1528 @classmethod
1529 def suitable(cls, url):
1530 """Receives a URL and returns True if suitable for this IE."""
1531 return re.match(cls._VALID_URL, url, re.VERBOSE) is not None
1532
1533 def _real_extract(self, url):
1534 # Extract playlist id
1535 mobj = re.match(self._VALID_URL, url, re.VERBOSE)
1536 if mobj is None:
1537 raise ExtractorError(u'Invalid URL: %s' % url)
1538
1539 # Download playlist videos from API
1540 playlist_id = mobj.group(1) or mobj.group(2)
1541 videos = []
1542
1543 for page_num in itertools.count(1):
1544 start_index = self._MAX_RESULTS * (page_num - 1) + 1
1545 if start_index >= 1000:
1546 self._downloader.report_warning(u'Max number of results reached')
1547 break
1548 url = self._TEMPLATE_URL % (playlist_id, self._MAX_RESULTS, start_index)
1549 page = self._download_webpage(url, playlist_id, u'Downloading page #%s' % page_num)
1550
1551 try:
1552 response = json.loads(page)
1553 except ValueError as err:
1554 raise ExtractorError(u'Invalid JSON in API response: ' + compat_str(err))
1555
1556 if 'feed' not in response:
1557 raise ExtractorError(u'Got a malformed response from YouTube API')
1558 playlist_title = response['feed']['title']['$t']
1559 if 'entry' not in response['feed']:
1560 # Number of videos is a multiple of self._MAX_RESULTS
1561 break
1562
1563 for entry in response['feed']['entry']:
1564 index = entry['yt$position']['$t']
1565 if 'media$group' in entry and 'yt$videoid' in entry['media$group']:
1566 videos.append((
1567 index,
1568 'https://www.youtube.com/watch?v=' + entry['media$group']['yt$videoid']['$t']
1569 ))
1570
1571 videos = [v[1] for v in sorted(videos)]
1572
1573 url_results = [self.url_result(vurl, 'Youtube') for vurl in videos]
1574 return [self.playlist_result(url_results, playlist_id, playlist_title)]
1575
1576
1577 class YoutubeChannelIE(InfoExtractor):
1578 IE_DESC = u'YouTube.com channels'
1579 _VALID_URL = r"^(?:https?://)?(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie)?\.com)/channel/([0-9A-Za-z_-]+)"
1580 _TEMPLATE_URL = 'http://www.youtube.com/channel/%s/videos?sort=da&flow=list&view=0&page=%s&gl=US&hl=en'
1581 _MORE_PAGES_INDICATOR = 'yt-uix-load-more'
1582 _MORE_PAGES_URL = 'http://www.youtube.com/c4_browse_ajax?action_load_more_videos=1&flow=list&paging=%s&view=0&sort=da&channel_id=%s'
1583 IE_NAME = u'youtube:channel'
1584
1585 def extract_videos_from_page(self, page):
1586 ids_in_page = []
1587 for mobj in re.finditer(r'href="/watch\?v=([0-9A-Za-z_-]+)&?', page):
1588 if mobj.group(1) not in ids_in_page:
1589 ids_in_page.append(mobj.group(1))
1590 return ids_in_page
1591
1592 def _real_extract(self, url):
1593 # Extract channel id
1594 mobj = re.match(self._VALID_URL, url)
1595 if mobj is None:
1596 raise ExtractorError(u'Invalid URL: %s' % url)
1597
1598 # Download channel page
1599 channel_id = mobj.group(1)
1600 video_ids = []
1601 pagenum = 1
1602
1603 url = self._TEMPLATE_URL % (channel_id, pagenum)
1604 page = self._download_webpage(url, channel_id,
1605 u'Downloading page #%s' % pagenum)
1606
1607 # Extract video identifiers
1608 ids_in_page = self.extract_videos_from_page(page)
1609 video_ids.extend(ids_in_page)
1610
1611 # Download any subsequent channel pages using the json-based channel_ajax query
1612 if self._MORE_PAGES_INDICATOR in page:
1613 for pagenum in itertools.count(1):
1614 url = self._MORE_PAGES_URL % (pagenum, channel_id)
1615 page = self._download_webpage(url, channel_id,
1616 u'Downloading page #%s' % pagenum)
1617
1618 page = json.loads(page)
1619
1620 ids_in_page = self.extract_videos_from_page(page['content_html'])
1621 video_ids.extend(ids_in_page)
1622
1623 if self._MORE_PAGES_INDICATOR not in page['load_more_widget_html']:
1624 break
1625
1626 self._downloader.to_screen(u'[youtube] Channel %s: Found %i videos' % (channel_id, len(video_ids)))
1627
1628 urls = ['http://www.youtube.com/watch?v=%s' % id for id in video_ids]
1629 url_entries = [self.url_result(eurl, 'Youtube') for eurl in urls]
1630 return [self.playlist_result(url_entries, channel_id)]
1631
1632
1633 class YoutubeUserIE(InfoExtractor):
1634 IE_DESC = u'YouTube.com user videos (URL or "ytuser" keyword)'
1635 _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:user/)?)|ytuser:)(?!feed/)([A-Za-z0-9_-]+)'
1636 _TEMPLATE_URL = 'http://gdata.youtube.com/feeds/api/users/%s'
1637 _GDATA_PAGE_SIZE = 50
1638 _GDATA_URL = 'http://gdata.youtube.com/feeds/api/users/%s/uploads?max-results=%d&start-index=%d&alt=json'
1639 IE_NAME = u'youtube:user'
1640
1641 @classmethod
1642 def suitable(cls, url):
1643 # Don't return True if the url can be extracted with other youtube
1644 # extractor, the regex would is too permissive and it would match.
1645 other_ies = iter(klass for (name, klass) in globals().items() if name.endswith('IE') and klass is not cls)
1646 if any(ie.suitable(url) for ie in other_ies): return False
1647 else: return super(YoutubeUserIE, cls).suitable(url)
1648
1649 def _real_extract(self, url):
1650 # Extract username
1651 mobj = re.match(self._VALID_URL, url)
1652 if mobj is None:
1653 raise ExtractorError(u'Invalid URL: %s' % url)
1654
1655 username = mobj.group(1)
1656
1657 # Download video ids using YouTube Data API. Result size per
1658 # query is limited (currently to 50 videos) so we need to query
1659 # page by page until there are no video ids - it means we got
1660 # all of them.
1661
1662 video_ids = []
1663
1664 for pagenum in itertools.count(0):
1665 start_index = pagenum * self._GDATA_PAGE_SIZE + 1
1666
1667 gdata_url = self._GDATA_URL % (username, self._GDATA_PAGE_SIZE, start_index)
1668 page = self._download_webpage(gdata_url, username,
1669 u'Downloading video ids from %d to %d' % (start_index, start_index + self._GDATA_PAGE_SIZE))
1670
1671 try:
1672 response = json.loads(page)
1673 except ValueError as err:
1674 raise ExtractorError(u'Invalid JSON in API response: ' + compat_str(err))
1675 if 'entry' not in response['feed']:
1676 # Number of videos is a multiple of self._MAX_RESULTS
1677 break
1678
1679 # Extract video identifiers
1680 ids_in_page = []
1681 for entry in response['feed']['entry']:
1682 ids_in_page.append(entry['id']['$t'].split('/')[-1])
1683 video_ids.extend(ids_in_page)
1684
1685 # A little optimization - if current page is not
1686 # "full", ie. does not contain PAGE_SIZE video ids then
1687 # we can assume that this page is the last one - there
1688 # are no more ids on further pages - no need to query
1689 # again.
1690
1691 if len(ids_in_page) < self._GDATA_PAGE_SIZE:
1692 break
1693
1694 urls = ['http://www.youtube.com/watch?v=%s' % video_id for video_id in video_ids]
1695 url_results = [self.url_result(rurl, 'Youtube') for rurl in urls]
1696 return [self.playlist_result(url_results, playlist_title = username)]
1697
1698 class YoutubeSearchIE(SearchInfoExtractor):
1699 IE_DESC = u'YouTube.com searches'
1700 _API_URL = 'https://gdata.youtube.com/feeds/api/videos?q=%s&start-index=%i&max-results=50&v=2&alt=jsonc'
1701 _MAX_RESULTS = 1000
1702 IE_NAME = u'youtube:search'
1703 _SEARCH_KEY = 'ytsearch'
1704
1705 def report_download_page(self, query, pagenum):
1706 """Report attempt to download search page with given number."""
1707 self._downloader.to_screen(u'[youtube] query "%s": Downloading page %s' % (query, pagenum))
1708
1709 def _get_n_results(self, query, n):
1710 """Get a specified number of results for a query"""
1711
1712 video_ids = []
1713 pagenum = 0
1714 limit = n
1715
1716 while (50 * pagenum) < limit:
1717 self.report_download_page(query, pagenum+1)
1718 result_url = self._API_URL % (compat_urllib_parse.quote_plus(query), (50*pagenum)+1)
1719 request = compat_urllib_request.Request(result_url)
1720 try:
1721 data = compat_urllib_request.urlopen(request).read().decode('utf-8')
1722 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1723 raise ExtractorError(u'Unable to download API page: %s' % compat_str(err))
1724 api_response = json.loads(data)['data']
1725
1726 if not 'items' in api_response:
1727 raise ExtractorError(u'[youtube] No video results')
1728
1729 new_ids = list(video['id'] for video in api_response['items'])
1730 video_ids += new_ids
1731
1732 limit = min(n, api_response['totalItems'])
1733 pagenum += 1
1734
1735 if len(video_ids) > n:
1736 video_ids = video_ids[:n]
1737 videos = [self.url_result('http://www.youtube.com/watch?v=%s' % id, 'Youtube') for id in video_ids]
1738 return self.playlist_result(videos, query)
1739
1740
1741 class YoutubeShowIE(InfoExtractor):
1742 IE_DESC = u'YouTube.com (multi-season) shows'
1743 _VALID_URL = r'https?://www\.youtube\.com/show/(.*)'
1744 IE_NAME = u'youtube:show'
1745
1746 def _real_extract(self, url):
1747 mobj = re.match(self._VALID_URL, url)
1748 show_name = mobj.group(1)
1749 webpage = self._download_webpage(url, show_name, u'Downloading show webpage')
1750 # There's one playlist for each season of the show
1751 m_seasons = list(re.finditer(r'href="(/playlist\?list=.*?)"', webpage))
1752 self.to_screen(u'%s: Found %s seasons' % (show_name, len(m_seasons)))
1753 return [self.url_result('https://www.youtube.com' + season.group(1), 'YoutubePlaylist') for season in m_seasons]
1754
1755
1756 class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor):
1757 """
1758 Base class for extractors that fetch info from
1759 http://www.youtube.com/feed_ajax
1760 Subclasses must define the _FEED_NAME and _PLAYLIST_TITLE properties.
1761 """
1762 _LOGIN_REQUIRED = True
1763 _PAGING_STEP = 30
1764 # use action_load_personal_feed instead of action_load_system_feed
1765 _PERSONAL_FEED = False
1766
1767 @property
1768 def _FEED_TEMPLATE(self):
1769 action = 'action_load_system_feed'
1770 if self._PERSONAL_FEED:
1771 action = 'action_load_personal_feed'
1772 return 'http://www.youtube.com/feed_ajax?%s=1&feed_name=%s&paging=%%s' % (action, self._FEED_NAME)
1773
1774 @property
1775 def IE_NAME(self):
1776 return u'youtube:%s' % self._FEED_NAME
1777
1778 def _real_initialize(self):
1779 self._login()
1780
1781 def _real_extract(self, url):
1782 feed_entries = []
1783 # The step argument is available only in 2.7 or higher
1784 for i in itertools.count(0):
1785 paging = i*self._PAGING_STEP
1786 info = self._download_webpage(self._FEED_TEMPLATE % paging,
1787 u'%s feed' % self._FEED_NAME,
1788 u'Downloading page %s' % i)
1789 info = json.loads(info)
1790 feed_html = info['feed_html']
1791 m_ids = re.finditer(r'"/watch\?v=(.*?)["&]', feed_html)
1792 ids = orderedSet(m.group(1) for m in m_ids)
1793 feed_entries.extend(self.url_result(id, 'Youtube') for id in ids)
1794 if info['paging'] is None:
1795 break
1796 return self.playlist_result(feed_entries, playlist_title=self._PLAYLIST_TITLE)
1797
1798 class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
1799 IE_DESC = u'YouTube.com subscriptions feed, "ytsubs" keyword(requires authentication)'
1800 _VALID_URL = r'https?://www\.youtube\.com/feed/subscriptions|:ytsubs(?:criptions)?'
1801 _FEED_NAME = 'subscriptions'
1802 _PLAYLIST_TITLE = u'Youtube Subscriptions'
1803
1804 class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
1805 IE_DESC = u'YouTube.com recommended videos, "ytrec" keyword (requires authentication)'
1806 _VALID_URL = r'https?://www\.youtube\.com/feed/recommended|:ytrec(?:ommended)?'
1807 _FEED_NAME = 'recommended'
1808 _PLAYLIST_TITLE = u'Youtube Recommended videos'
1809
1810 class YoutubeWatchLaterIE(YoutubeFeedsInfoExtractor):
1811 IE_DESC = u'Youtube watch later list, "ytwatchlater" keyword (requires authentication)'
1812 _VALID_URL = r'https?://www\.youtube\.com/feed/watch_later|:ytwatchlater'
1813 _FEED_NAME = 'watch_later'
1814 _PLAYLIST_TITLE = u'Youtube Watch Later'
1815 _PAGING_STEP = 100
1816 _PERSONAL_FEED = True
1817
1818 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
1819 IE_NAME = u'youtube:favorites'
1820 IE_DESC = u'YouTube.com favourite videos, "ytfav" keyword (requires authentication)'
1821 _VALID_URL = r'https?://www\.youtube\.com/my_favorites|:ytfav(?:ou?rites)?'
1822 _LOGIN_REQUIRED = True
1823
1824 def _real_extract(self, url):
1825 webpage = self._download_webpage('https://www.youtube.com/my_favorites', 'Youtube Favourites videos')
1826 playlist_id = self._search_regex(r'list=(.+?)["&]', webpage, u'favourites playlist id')
1827 return self.url_result(playlist_id, 'YoutubePlaylist')
fork of https://github.com/yt-dlp/yt-dlp/