19 _NAME_RE
= r
'[a-zA-Z_$][\w$]*'
21 # Ref: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Operator_Precedence
22 _OPERATORS
= { # None => Defined in JSInterpreter._operator
27 '&': lambda a
, b
: (a
or 0) & (b
or 0),
28 '|': lambda a
, b
: (a
or 0) |
(b
or 0),
29 '^': lambda a
, b
: (a
or 0) ^
(b
or 0),
32 '!==': operator
.is_not
,
36 '<=': lambda a
, b
: (a
or 0) <= (b
or 0),
37 '>=': lambda a
, b
: (a
or 0) >= (b
or 0),
38 '<': lambda a
, b
: (a
or 0) < (b
or 0),
39 '>': lambda a
, b
: (a
or 0) > (b
or 0),
41 '>>': operator
.rshift
,
42 '<<': operator
.lshift
,
44 '+': lambda a
, b
: (a
or 0) + (b
or 0),
45 '-': lambda a
, b
: (a
or 0) - (b
or 0),
47 '*': lambda a
, b
: (a
or 0) * (b
or 0),
48 '/': lambda a
, b
: (a
or 0) / b
if b
else float('NaN'),
49 '%': lambda a
, b
: (a
or 0) % b
if b
else float('NaN'),
54 _COMP_OPERATORS
= {'===', '!==', '==', '!=', '<=', '>=', '<', '>'}
56 _MATCHING_PARENS
= dict(zip('({[', ')}]'))
60 def _ternary(cndn
, if_true
=True, if_false
=False):
61 """Simulate JS's ternary operator (cndn?if_true:if_false)"""
62 if cndn
in (False, None, 0, ''):
64 with contextlib
.suppress(TypeError):
65 if math
.isnan(cndn
): # NB: NaN cannot be checked by membership
70 class JS_Break(ExtractorError
):
72 ExtractorError
.__init
__(self
, 'Invalid break')
75 class JS_Continue(ExtractorError
):
77 ExtractorError
.__init
__(self
, 'Invalid continue')
80 class JS_Throw(ExtractorError
):
81 def __init__(self
, e
):
83 ExtractorError
.__init
__(self
, f
'Uncaught exception {e}')
86 class LocalNameSpace(collections
.ChainMap
):
87 def __setitem__(self
, key
, value
):
88 for scope
in self
.maps
:
92 self
.maps
[0][key
] = value
94 def __delitem__(self
, key
):
95 raise NotImplementedError('Deleting is not supported')
100 ENABLED
= False and 'pytest' in sys
.modules
103 def write(*args
, level
=100):
104 write_string(f
'[debug] JS: {" " * (100 - level)}'
105 f
'{" ".join(truncate_string(str(x), 50, 50) for x in args)}\n')
108 def wrap_interpreter(cls
, f
):
109 def interpret_statement(self
, stmt
, local_vars
, allow_recursion
, *args
, **kwargs
):
110 if cls
.ENABLED
and stmt
.strip():
111 cls
.write(stmt
, level
=allow_recursion
)
112 ret
, should_ret
= f(self
, stmt
, local_vars
, allow_recursion
, *args
, **kwargs
)
113 if cls
.ENABLED
and stmt
.strip():
114 cls
.write(['->', '=>'][should_ret
], repr(ret
), '<-|', stmt
, level
=allow_recursion
)
115 return ret
, should_ret
116 return interpret_statement
120 __named_object_counter
= 0
122 def __init__(self
, code
, objects
=None):
123 self
.code
, self
._functions
= code
, {}
124 self
._objects
= {} if objects
is None else objects
126 class Exception(ExtractorError
):
127 def __init__(self
, msg
, expr
=None, *args
, **kwargs
):
129 msg
= f
'{msg.rstrip()} in: {truncate_string(expr, 50, 50)}'
130 super().__init
__(msg
, *args
, **kwargs
)
132 def _named_object(self
, namespace
, obj
):
133 self
.__named
_object
_counter
+= 1
134 name
= f
'__yt_dlp_jsinterp_obj{self.__named_object_counter}'
135 namespace
[name
] = obj
139 def _separate(expr
, delim
=',', max_split
=None):
140 OP_CHARS
= '+-*/%&|^=<>!,;'
143 counters
= {k: 0 for k in _MATCHING_PARENS.values()}
144 start
, splits
, pos
, delim_len
= 0, 0, 0, len(delim
) - 1
145 in_quote
, escaping
, after_op
, in_regex_char_group
= None, False, True, False
146 for idx
, char
in enumerate(expr
):
147 if not in_quote
and char
in _MATCHING_PARENS
:
148 counters
[_MATCHING_PARENS
[char
]] += 1
149 elif not in_quote
and char
in counters
:
151 elif not escaping
and char
in _QUOTES
and in_quote
in (char
, None):
152 if in_quote
or after_op
or char
!= '/':
153 in_quote
= None if in_quote
and not in_regex_char_group
else char
154 elif in_quote
== '/' and char
in '[]':
155 in_regex_char_group
= char
== '['
156 escaping
= not escaping
and in_quote
and char
== '\\'
157 after_op
= not in_quote
and char
in OP_CHARS
or (char
== ' ' and after_op
)
159 if char
!= delim
[pos
] or any(counters
.values()) or in_quote
:
162 elif pos
!= delim_len
:
165 yield expr
[start
: idx
- delim_len
]
166 start
, pos
= idx
+ 1, 0
168 if max_split
and splits
>= max_split
:
173 def _separate_at_paren(cls
, expr
, delim
):
174 separated
= list(cls
._separate
(expr
, delim
, 1))
175 if len(separated
) < 2:
176 raise cls
.Exception(f
'No terminating paren {delim}', expr
)
177 return separated
[0][1:].strip(), separated
[1].strip()
179 def _operator(self
, op
, left_val
, right_expr
, expr
, local_vars
, allow_recursion
):
180 if op
in ('||', '&&'):
181 if (op
== '&&') ^
_ternary(left_val
):
182 return left_val
# short circuiting
184 right_expr
= _ternary(left_val
, *self
._separate
(right_expr
, ':', 1))
186 right_val
= self
.interpret_expression(right_expr
, local_vars
, allow_recursion
)
187 if not _OPERATORS
.get(op
):
191 return _OPERATORS
[op
](left_val
, right_val
)
192 except Exception as e
:
193 raise self
.Exception(f
'Failed to evaluate {left_val!r} {op} {right_val!r}', expr
, cause
=e
)
195 def _index(self
, obj
, idx
):
199 return obj
[int(idx
)] if isinstance(obj
, list) else obj
[idx
]
200 except Exception as e
:
201 raise self
.Exception(f
'Cannot get index {idx}', repr(obj
), cause
=e
)
203 def _dump(self
, obj
, namespace
):
205 return json
.dumps(obj
)
207 return self
._named
_object
(namespace
, obj
)
209 @Debugger.wrap_interpreter
210 def interpret_statement(self
, stmt
, local_vars
, allow_recursion
=100):
211 if allow_recursion
< 0:
212 raise self
.Exception('Recursion limit reached')
215 should_return
= False
216 sub_statements
= list(self
._separate
(stmt
, ';')) or ['']
217 expr
= stmt
= sub_statements
.pop().strip()
219 for sub_stmt
in sub_statements
:
220 ret
, should_return
= self
.interpret_statement(sub_stmt
, local_vars
, allow_recursion
)
222 return ret
, should_return
224 m
= re
.match(r
'(?P<var>(?:var|const|let)\s)|return(?:\s+|(?=["\'])|$
)|
(?P
<throw
>throw\s
+)', stmt)
226 expr = stmt[len(m.group(0)):].strip()
228 raise JS_Throw(self.interpret_expression(expr, local_vars, allow_recursion))
229 should_return = not m.group('var
')
231 return None, should_return
233 if expr[0] in _QUOTES:
234 inner, outer = self._separate(expr, expr[0], 1)
236 inner = inner[1:].replace('"', R'\"')
237 inner = re.compile(json.loads(js_to_json(f'"{inner}
"', strict=True)))
239 inner = json.loads(js_to_json(f'{inner}{expr[0]}', strict=True))
241 return inner, should_return
242 expr = self._named_object(local_vars, inner) + outer
244 if expr.startswith('new '):
246 if obj.startswith('Date('):
247 left, right = self._separate_at_paren(obj[4:], ')')
248 expr = unified_timestamp(
249 self.interpret_expression(left, local_vars, allow_recursion), False)
251 raise self.Exception(f'Failed to parse date {left!r}', expr)
252 expr = self._dump(int(expr * 1000), local_vars) + right
254 raise self.Exception(f'Unsupported object {obj}', expr)
256 if expr.startswith('void '):
257 left = self.interpret_expression(expr[5:], local_vars, allow_recursion)
258 return None, should_return
260 if expr.startswith('{'):
261 inner, outer = self._separate_at_paren(expr, '}')
262 inner, should_abort = self.interpret_statement(inner, local_vars, allow_recursion)
263 if not outer or should_abort:
264 return inner, should_abort or should_return
266 expr = self._dump(inner, local_vars) + outer
268 if expr.startswith('('):
269 inner, outer = self._separate_at_paren(expr, ')')
270 inner, should_abort = self.interpret_statement(inner, local_vars, allow_recursion)
271 if not outer or should_abort:
272 return inner, should_abort or should_return
274 expr = self._dump(inner, local_vars) + outer
276 if expr.startswith('['):
277 inner, outer = self._separate_at_paren(expr, ']')
278 name = self._named_object(local_vars, [
279 self.interpret_expression(item, local_vars, allow_recursion)
280 for item in self._separate(inner)])
283 m = re.match(rf'''(?x)
284 (?P<try>try|finally)\s*|
285 (?P<catch>catch\s*(?P<err>\(\s*{_NAME_RE}\s*\)))|
286 (?P<switch>switch)\s*\(|
287 (?P<for>for)\s*\(|''', expr)
288 if m and m.group('try'):
289 if expr[m.end()] == '{':
290 try_expr, expr = self._separate_at_paren(expr[m.end():], '}')
292 try_expr, expr = expr[m.end() - 1:], ''
294 ret, should_abort = self.interpret_statement(try_expr, local_vars, allow_recursion)
297 except JS_Throw as e:
298 local_vars['__ytdlp_exception__'] = e.error
299 except Exception as e:
300 # XXX: This works for now, but makes debugging future issues very hard
301 local_vars['__ytdlp_exception__'] = e
302 ret, should_abort = self.interpret_statement(expr, local_vars, allow_recursion)
303 return ret, should_abort or should_return
305 elif m and m.group('catch'):
306 catch_expr, expr = self._separate_at_paren(expr[m.end():], '}')
307 if '__ytdlp_exception__' in local_vars:
308 catch_vars = local_vars.new_child({m.group('err'): local_vars.pop('__ytdlp_exception__')})
309 ret, should_abort = self.interpret_statement(catch_expr, catch_vars, allow_recursion)
313 ret, should_abort = self.interpret_statement(expr, local_vars, allow_recursion)
314 return ret, should_abort or should_return
316 elif m and m.group('for'):
317 constructor, remaining = self._separate_at_paren(expr[m.end() - 1:], ')')
318 if remaining.startswith('{'):
319 body, expr = self._separate_at_paren(remaining, '}')
321 switch_m = re.match(r'switch\s*\(', remaining) # FIXME
323 switch_val, remaining = self._separate_at_paren(remaining[switch_m.end() - 1:], ')')
324 body, expr = self._separate_at_paren(remaining, '}')
325 body = 'switch(%s){%s}' % (switch_val, body)
327 body, expr = remaining, ''
328 start, cndn, increment = self._separate(constructor, ';')
329 self.interpret_expression(start, local_vars, allow_recursion)
331 if not _ternary(self.interpret_expression(cndn, local_vars, allow_recursion)):
334 ret, should_abort = self.interpret_statement(body, local_vars, allow_recursion)
341 self.interpret_expression(increment, local_vars, allow_recursion)
342 ret, should_abort = self.interpret_statement(expr, local_vars, allow_recursion)
343 return ret, should_abort or should_return
345 elif m and m.group('switch'):
346 switch_val, remaining = self._separate_at_paren(expr[m.end() - 1:], ')')
347 switch_val = self.interpret_expression(switch_val, local_vars, allow_recursion)
348 body, expr = self._separate_at_paren(remaining, '}')
349 items = body.replace('default:', 'case default:').split('case ')[1:]
350 for default in (False, True):
353 case, stmt = (i.strip() for i in self._separate(item, ':', 1))
355 matched = matched or case == 'default'
357 matched = (case != 'default'
358 and switch_val == self.interpret_expression(case, local_vars, allow_recursion))
362 ret, should_abort = self.interpret_statement(stmt, local_vars, allow_recursion)
369 ret, should_abort = self.interpret_statement(expr, local_vars, allow_recursion)
370 return ret, should_abort or should_return
372 # Comma separated statements
373 sub_expressions = list(self._separate(expr))
374 if len(sub_expressions) > 1:
375 for sub_expr in sub_expressions:
376 ret, should_abort = self.interpret_statement(sub_expr, local_vars, allow_recursion)
381 for m in re.finditer(rf'''(?x)
382 (?P<pre_sign>\+\+|--)(?P<var1>{_NAME_RE})|
383 (?P<var2>{_NAME_RE})(?P<post_sign>\+\+|--)''', expr):
384 var = m.group('var1') or m.group('var2')
385 start, end = m.span()
386 sign = m.group('pre_sign') or m.group('post_sign')
387 ret = local_vars[var]
388 local_vars[var] += 1 if sign[0] == '+' else -1
389 if m.group('pre_sign'):
390 ret = local_vars[var]
391 expr = expr[:start] + self._dump(ret, local_vars) + expr[end:]
394 return None, should_return
396 m = re.match(fr'''(?x)
398 (?P<out>{_NAME_RE})(?:\[(?P<index>[^\]]+?)\])?\s*
399 (?P<op>{"|".join(map(re.escape, set(_OPERATORS) - _COMP_OPERATORS))})?
402 (?!if|return|true|false|null|undefined)(?P<name>{_NAME_RE})$
404 (?P<in>{_NAME_RE})\[(?P<idx>.+)\]$
406 (?P<var>{_NAME_RE})(?:\.(?P<member>[^(]+)|\[(?P<member2>[^\]]+)\])\s*
408 (?P<fname>{_NAME_RE})\((?P<args>.*)\)$
410 if m and m.group('assign'):
411 left_val = local_vars.get(m.group('out'))
413 if not m.group('index'):
414 local_vars[m.group('out')] = self._operator(
415 m.group('op'), left_val, m.group('expr'), expr, local_vars, allow_recursion)
416 return local_vars[m.group('out')], should_return
417 elif left_val is None:
418 raise self.Exception(f'Cannot index undefined variable {m.group("out")}', expr)
420 idx = self.interpret_expression(m.group('index'), local_vars, allow_recursion)
421 if not isinstance(idx, (int, float)):
422 raise self.Exception(f'List index {idx} must be integer', expr)
424 left_val[idx] = self._operator(
425 m.group('op'), self._index(left_val, idx), m.group('expr'), expr, local_vars, allow_recursion)
426 return left_val[idx], should_return
429 return int(expr), should_return
431 elif expr == 'break':
433 elif expr == 'continue':
436 elif m and m.group('return'):
437 return local_vars[m.group('name')], should_return
439 with contextlib.suppress(ValueError):
440 return json.loads(js_to_json(expr, strict=True)), should_return
442 if m and m.group('indexing'):
443 val = local_vars[m.group('in')]
444 idx = self.interpret_expression(m.group('idx'), local_vars, allow_recursion)
445 return self._index(val, idx), should_return
447 for op in _OPERATORS:
448 separated = list(self._separate(expr, op))
449 right_expr = separated.pop()
450 while op in '<>*-' and len(separated) > 1 and not separated[-1].strip():
452 right_expr = f'{op}{right_expr}'
454 right_expr = f'{separated.pop()}{op}{right_expr}'
457 left_val = self.interpret_expression(op.join(separated), local_vars, allow_recursion)
458 return self._operator(op, left_val, right_expr, expr, local_vars, allow_recursion), should_return
460 if m and m.group('attribute'):
461 variable = m.group('var')
462 member = m.group('member')
464 member = self.interpret_expression(m.group('member2'), local_vars, allow_recursion)
465 arg_str = expr[m.end():]
466 if arg_str.startswith('('):
467 arg_str, remaining = self._separate_at_paren(arg_str, ')')
469 arg_str, remaining = None, arg_str
471 def assertion(cndn, msg):
472 """ assert, but without risk of getting optimized out """
474 raise self.Exception(f'{member} {msg}', expr)
477 if (variable, member) == ('console', 'debug'):
479 Debugger.write(self.interpret_expression(f'[{arg_str}]', local_vars, allow_recursion))
486 obj = local_vars.get(variable, types.get(variable, NO_DEFAULT))
487 if obj is NO_DEFAULT:
488 if variable not in self._objects:
489 self._objects[variable] = self.extract_object(variable)
490 obj = self._objects[variable]
494 return self._index(obj, member)
498 self.interpret_expression(v, local_vars, allow_recursion)
499 for v in self._separate(arg_str)]
502 if member == 'fromCharCode':
503 assertion(argvals, 'takes one or more arguments')
504 return ''.join(map(chr, argvals))
505 raise self.Exception(f'Unsupported String method {member}', expr)
508 assertion(len(argvals) == 2, 'takes two arguments')
509 return argvals[0] ** argvals[1]
510 raise self.Exception(f'Unsupported Math method {member}', expr)
512 if member == 'split':
513 assertion(argvals, 'takes one or more arguments')
514 assertion(len(argvals) == 1, 'with limit argument is not implemented')
515 return obj.split(argvals[0]) if argvals[0] else list(obj)
516 elif member == 'join':
517 assertion(isinstance(obj, list), 'must be applied on a list')
518 assertion(len(argvals) == 1, 'takes exactly one argument')
519 return argvals[0].join(obj)
520 elif member == 'reverse':
521 assertion(not argvals, 'does not take any arguments')
524 elif member == 'slice':
525 assertion(isinstance(obj, list), 'must be applied on a list')
526 assertion(len(argvals) == 1, 'takes exactly one argument')
527 return obj[argvals[0]:]
528 elif member == 'splice':
529 assertion(isinstance(obj, list), 'must be applied on a list')
530 assertion(argvals, 'takes one or more arguments')
531 index, howMany = map(int, (argvals + [len(obj)])[:2])
534 add_items = argvals[2:]
536 for i in range(index, min(index + howMany, len(obj))):
537 res.append(obj.pop(index))
538 for i, item in enumerate(add_items):
539 obj.insert(index + i, item)
541 elif member == 'unshift':
542 assertion(isinstance(obj, list), 'must be applied on a list')
543 assertion(argvals, 'takes one or more arguments')
544 for item in reversed(argvals):
547 elif member == 'pop':
548 assertion(isinstance(obj, list), 'must be applied on a list')
549 assertion(not argvals, 'does not take any arguments')
553 elif member == 'push':
554 assertion(argvals, 'takes one or more arguments')
557 elif member == 'forEach':
558 assertion(argvals, 'takes one or more arguments')
559 assertion(len(argvals) <= 2, 'takes at-most 2 arguments')
560 f, this = (argvals + [''])[:2]
561 return [f((item, idx, obj), {'this': this}, allow_recursion) for idx, item in enumerate(obj)]
562 elif member == 'indexOf':
563 assertion(argvals, 'takes one or more arguments')
564 assertion(len(argvals) <= 2, 'takes at-most 2 arguments')
565 idx, start = (argvals + [0])[:2]
567 return obj.index(idx, start)
571 idx = int(member) if isinstance(obj, list) else member
572 return obj[idx](argvals, allow_recursion=allow_recursion)
575 ret, should_abort = self.interpret_statement(
576 self._named_object(local_vars, eval_method()) + remaining,
577 local_vars, allow_recursion)
578 return ret, should_return or should_abort
580 return eval_method(), should_return
582 elif m and m.group('function'):
583 fname = m.group('fname')
584 argvals = [self.interpret_expression(v, local_vars, allow_recursion)
585 for v in self._separate(m.group('args'))]
586 if fname in local_vars:
587 return local_vars[fname](argvals, allow_recursion=allow_recursion), should_return
588 elif fname not in self._functions:
589 self._functions[fname] = self.extract_function(fname)
590 return self._functions[fname](argvals, allow_recursion=allow_recursion), should_return
592 raise self.Exception(
593 f'Unsupported JS expression {truncate_string(expr, 20, 20) if expr != stmt else ""}', stmt)
595 def interpret_expression(self, expr, local_vars, allow_recursion):
596 ret, should_return = self.interpret_statement(expr, local_vars, allow_recursion)
598 raise self.Exception('Cannot return from an expression', expr)
601 def extract_object(self, objname):
602 _FUNC_NAME_RE = r'''(?:[a-zA-Z$0-9]+|"[a
-zA
-Z$
0-9]+"|'[a-zA-Z$0-9]+')'''
606 (?<!this\.)%s\s*=\s*{\s*
607 (?P<fields>(%s\s*:\s*function\s*\(.*?\)\s*{.*?}(?:,\s*)?)*)
609 ''' % (re.escape(objname), _FUNC_NAME_RE),
612 raise self.Exception(f'Could not find object {objname}')
613 fields = obj_m.group('fields')
614 # Currently, it only supports function definitions
615 fields_m = re.finditer(
617 (?P<key>%s)\s*:\s*function\s*\((?P<args>(?:%s|,)*)\){(?P<code>[^}]+)}
618 ''' % (_FUNC_NAME_RE, _NAME_RE),
621 argnames = f.group('args').split(',')
622 obj[remove_quotes(f.group('key'))] = self.build_function(argnames, f.group('code'))
626 def extract_function_code(self, funcname):
627 """ @returns argnames, code """
632 [{;,]\s*%(name)s\s*=\s*function|
633 (?:var|const|let)\s+%(name)s\s*=\s*function
635 \((?P<args>[^)]*)\)\s*
636 (?P<code>{.+})''' % {'name': re.escape(funcname)},
638 code, _ = self._separate_at_paren(func_m.group('code'), '}')
640 raise self.Exception(f'Could not find JS function "{funcname}
"')
641 return [x.strip() for x in func_m.group('args').split(',')], code
643 def extract_function(self, funcname):
644 return self.extract_function_from_code(*self.extract_function_code(funcname))
646 def extract_function_from_code(self, argnames, code, *global_stack):
649 mobj = re.search(r'function\((?P<args>[^)]*)\)\s*{', code)
652 start, body_start = mobj.span()
653 body, remaining = self._separate_at_paren(code[body_start - 1:], '}')
654 name = self._named_object(local_vars, self.extract_function_from_code(
655 [x.strip() for x in mobj.group('args').split(',')],
656 body, local_vars, *global_stack))
657 code = code[:start] + name + remaining
658 return self.build_function(argnames, code, local_vars, *global_stack)
660 def call_function(self, funcname, *args):
661 return self.extract_function(funcname)(args)
663 def build_function(self, argnames, code, *global_stack):
664 global_stack = list(global_stack) or [{}]
665 argnames = tuple(argnames)
667 def resf(args, kwargs={}, allow_recursion=100):
668 global_stack[0].update(itertools.zip_longest(argnames, args, fillvalue=None))
669 global_stack[0].update(kwargs)
670 var_stack = LocalNameSpace(*global_stack)
671 ret, should_abort = self.interpret_statement(code.replace('\n', ''), var_stack, allow_recursion - 1)