- def test_traverse_obj(self):
- _TEST_DATA = {
- 100: 100,
- 1.2: 1.2,
- 'str': 'str',
- 'None': None,
- '...': ...,
- 'urls': [
- {'index': 0, 'url': 'https://www.example.com/0'},
- {'index': 1, 'url': 'https://www.example.com/1'},
- ],
- 'data': (
- {'index': 2},
- {'index': 3},
- ),
- 'dict': {},
- }
-
- # Test base functionality
- self.assertEqual(traverse_obj(_TEST_DATA, ('str',)), 'str',
- msg='allow tuple path')
- self.assertEqual(traverse_obj(_TEST_DATA, ['str']), 'str',
- msg='allow list path')
- self.assertEqual(traverse_obj(_TEST_DATA, (value for value in ("str",))), 'str',
- msg='allow iterable path')
- self.assertEqual(traverse_obj(_TEST_DATA, 'str'), 'str',
- msg='single items should be treated as a path')
- self.assertEqual(traverse_obj(_TEST_DATA, None), _TEST_DATA)
- self.assertEqual(traverse_obj(_TEST_DATA, 100), 100)
- self.assertEqual(traverse_obj(_TEST_DATA, 1.2), 1.2)
-
- # Test Ellipsis behavior
- self.assertCountEqual(traverse_obj(_TEST_DATA, ...),
- (item for item in _TEST_DATA.values() if item not in (None, {})),
- msg='`...` should give all non discarded values')
- self.assertCountEqual(traverse_obj(_TEST_DATA, ('urls', 0, ...)), _TEST_DATA['urls'][0].values(),
- msg='`...` selection for dicts should select all values')
- self.assertEqual(traverse_obj(_TEST_DATA, (..., ..., 'url')),
- ['https://www.example.com/0', 'https://www.example.com/1'],
- msg='nested `...` queries should work')
- self.assertCountEqual(traverse_obj(_TEST_DATA, (..., ..., 'index')), range(4),
- msg='`...` query result should be flattened')
- self.assertEqual(traverse_obj(iter(range(4)), ...), list(range(4)),
- msg='`...` should accept iterables')
-
- # Test function as key
- self.assertEqual(traverse_obj(_TEST_DATA, lambda x, y: x == 'urls' and isinstance(y, list)),
- [_TEST_DATA['urls']],
- msg='function as query key should perform a filter based on (key, value)')
- self.assertCountEqual(traverse_obj(_TEST_DATA, lambda _, x: isinstance(x[0], str)), {'str'},
- msg='exceptions in the query function should be catched')
- self.assertEqual(traverse_obj(iter(range(4)), lambda _, x: x % 2 == 0), [0, 2],
- msg='function key should accept iterables')
- if __debug__:
- with self.assertRaises(Exception, msg='Wrong function signature should raise in debug'):
- traverse_obj(_TEST_DATA, lambda a: ...)
- with self.assertRaises(Exception, msg='Wrong function signature should raise in debug'):
- traverse_obj(_TEST_DATA, lambda a, b, c: ...)
-
- # Test set as key (transformation/type, like `expected_type`)
- self.assertEqual(traverse_obj(_TEST_DATA, (..., {str.upper}, )), ['STR'],
- msg='Function in set should be a transformation')
- self.assertEqual(traverse_obj(_TEST_DATA, (..., {str})), ['str'],
- msg='Type in set should be a type filter')
- self.assertEqual(traverse_obj(_TEST_DATA, {dict}), _TEST_DATA,
- msg='A single set should be wrapped into a path')
- self.assertEqual(traverse_obj(_TEST_DATA, (..., {str.upper})), ['STR'],
- msg='Transformation function should not raise')
- self.assertEqual(traverse_obj(_TEST_DATA, (..., {str_or_none})),
- [item for item in map(str_or_none, _TEST_DATA.values()) if item is not None],
- msg='Function in set should be a transformation')
- if __debug__:
- with self.assertRaises(Exception, msg='Sets with length != 1 should raise in debug'):
- traverse_obj(_TEST_DATA, set())
- with self.assertRaises(Exception, msg='Sets with length != 1 should raise in debug'):
- traverse_obj(_TEST_DATA, {str.upper, str})
-
- # Test `slice` as a key
- _SLICE_DATA = [0, 1, 2, 3, 4]
- self.assertEqual(traverse_obj(_TEST_DATA, ('dict', slice(1))), None,
- msg='slice on a dictionary should not throw')
- self.assertEqual(traverse_obj(_SLICE_DATA, slice(1)), _SLICE_DATA[:1],
- msg='slice key should apply slice to sequence')
- self.assertEqual(traverse_obj(_SLICE_DATA, slice(1, 2)), _SLICE_DATA[1:2],
- msg='slice key should apply slice to sequence')
- self.assertEqual(traverse_obj(_SLICE_DATA, slice(1, 4, 2)), _SLICE_DATA[1:4:2],
- msg='slice key should apply slice to sequence')
-
- # Test alternative paths
- self.assertEqual(traverse_obj(_TEST_DATA, 'fail', 'str'), 'str',
- msg='multiple `paths` should be treated as alternative paths')
- self.assertEqual(traverse_obj(_TEST_DATA, 'str', 100), 'str',
- msg='alternatives should exit early')
- self.assertEqual(traverse_obj(_TEST_DATA, 'fail', 'fail'), None,
- msg='alternatives should return `default` if exhausted')
- self.assertEqual(traverse_obj(_TEST_DATA, (..., 'fail'), 100), 100,
- msg='alternatives should track their own branching return')
- self.assertEqual(traverse_obj(_TEST_DATA, ('dict', ...), ('data', ...)), list(_TEST_DATA['data']),
- msg='alternatives on empty objects should search further')
-
- # Test branch and path nesting
- self.assertEqual(traverse_obj(_TEST_DATA, ('urls', (3, 0), 'url')), ['https://www.example.com/0'],
- msg='tuple as key should be treated as branches')
- self.assertEqual(traverse_obj(_TEST_DATA, ('urls', [3, 0], 'url')), ['https://www.example.com/0'],
- msg='list as key should be treated as branches')
- self.assertEqual(traverse_obj(_TEST_DATA, ('urls', ((1, 'fail'), (0, 'url')))), ['https://www.example.com/0'],
- msg='double nesting in path should be treated as paths')
- self.assertEqual(traverse_obj(['0', [1, 2]], [(0, 1), 0]), [1],
- msg='do not fail early on branching')
- self.assertCountEqual(traverse_obj(_TEST_DATA, ('urls', ((1, ('fail', 'url')), (0, 'url')))),
- ['https://www.example.com/0', 'https://www.example.com/1'],
- msg='tripple nesting in path should be treated as branches')
- self.assertEqual(traverse_obj(_TEST_DATA, ('urls', ('fail', (..., 'url')))),
- ['https://www.example.com/0', 'https://www.example.com/1'],
- msg='ellipsis as branch path start gets flattened')
-
- # Test dictionary as key
- self.assertEqual(traverse_obj(_TEST_DATA, {0: 100, 1: 1.2}), {0: 100, 1: 1.2},
- msg='dict key should result in a dict with the same keys')
- self.assertEqual(traverse_obj(_TEST_DATA, {0: ('urls', 0, 'url')}),
- {0: 'https://www.example.com/0'},
- msg='dict key should allow paths')
- self.assertEqual(traverse_obj(_TEST_DATA, {0: ('urls', (3, 0), 'url')}),
- {0: ['https://www.example.com/0']},
- msg='tuple in dict path should be treated as branches')
- self.assertEqual(traverse_obj(_TEST_DATA, {0: ('urls', ((1, 'fail'), (0, 'url')))}),
- {0: ['https://www.example.com/0']},
- msg='double nesting in dict path should be treated as paths')
- self.assertEqual(traverse_obj(_TEST_DATA, {0: ('urls', ((1, ('fail', 'url')), (0, 'url')))}),
- {0: ['https://www.example.com/1', 'https://www.example.com/0']},
- msg='tripple nesting in dict path should be treated as branches')
- self.assertEqual(traverse_obj(_TEST_DATA, {0: 'fail'}), {},
- msg='remove `None` values when top level dict key fails')
- self.assertEqual(traverse_obj(_TEST_DATA, {0: 'fail'}, default=...), {0: ...},
- msg='use `default` if key fails and `default`')
- self.assertEqual(traverse_obj(_TEST_DATA, {0: 'dict'}), {},
- msg='remove empty values when dict key')
- self.assertEqual(traverse_obj(_TEST_DATA, {0: 'dict'}, default=...), {0: ...},
- msg='use `default` when dict key and `default`')
- self.assertEqual(traverse_obj(_TEST_DATA, {0: {0: 'fail'}}), {},
- msg='remove empty values when nested dict key fails')
- self.assertEqual(traverse_obj(None, {0: 'fail'}), {},
- msg='default to dict if pruned')
- self.assertEqual(traverse_obj(None, {0: 'fail'}, default=...), {0: ...},
- msg='default to dict if pruned and default is given')
- self.assertEqual(traverse_obj(_TEST_DATA, {0: {0: 'fail'}}, default=...), {0: {0: ...}},
- msg='use nested `default` when nested dict key fails and `default`')
- self.assertEqual(traverse_obj(_TEST_DATA, {0: ('dict', ...)}), {},
- msg='remove key if branch in dict key not successful')
-
- # Testing default parameter behavior
- _DEFAULT_DATA = {'None': None, 'int': 0, 'list': []}
- self.assertEqual(traverse_obj(_DEFAULT_DATA, 'fail'), None,
- msg='default value should be `None`')
- self.assertEqual(traverse_obj(_DEFAULT_DATA, 'fail', 'fail', default=...), ...,
- msg='chained fails should result in default')
- self.assertEqual(traverse_obj(_DEFAULT_DATA, 'None', 'int'), 0,
- msg='should not short cirquit on `None`')
- self.assertEqual(traverse_obj(_DEFAULT_DATA, 'fail', default=1), 1,
- msg='invalid dict key should result in `default`')
- self.assertEqual(traverse_obj(_DEFAULT_DATA, 'None', default=1), 1,
- msg='`None` is a deliberate sentinel and should become `default`')
- self.assertEqual(traverse_obj(_DEFAULT_DATA, ('list', 10)), None,
- msg='`IndexError` should result in `default`')
- self.assertEqual(traverse_obj(_DEFAULT_DATA, (..., 'fail'), default=1), 1,
- msg='if branched but not successful return `default` if defined, not `[]`')
- self.assertEqual(traverse_obj(_DEFAULT_DATA, (..., 'fail'), default=None), None,
- msg='if branched but not successful return `default` even if `default` is `None`')
- self.assertEqual(traverse_obj(_DEFAULT_DATA, (..., 'fail')), [],
- msg='if branched but not successful return `[]`, not `default`')
- self.assertEqual(traverse_obj(_DEFAULT_DATA, ('list', ...)), [],
- msg='if branched but object is empty return `[]`, not `default`')
- self.assertEqual(traverse_obj(None, ...), [],
- msg='if branched but object is `None` return `[]`, not `default`')
- self.assertEqual(traverse_obj({0: None}, (0, ...)), [],
- msg='if branched but state is `None` return `[]`, not `default`')
-
- branching_paths = [
- ('fail', ...),
- (..., 'fail'),
- 100 * ('fail',) + (...,),
- (...,) + 100 * ('fail',),
+ def test_try_call(self):
+ def total(*x, **kwargs):
+ return sum(x) + sum(kwargs.values())
+
+ self.assertEqual(try_call(None), None,
+ msg='not a fn should give None')
+ self.assertEqual(try_call(lambda: 1), 1,
+ msg='int fn with no expected_type should give int')
+ self.assertEqual(try_call(lambda: 1, expected_type=int), 1,
+ msg='int fn with expected_type int should give int')
+ self.assertEqual(try_call(lambda: 1, expected_type=dict), None,
+ msg='int fn with wrong expected_type should give None')
+ self.assertEqual(try_call(total, args=(0, 1, 0), expected_type=int), 1,
+ msg='fn should accept arglist')
+ self.assertEqual(try_call(total, kwargs={'a': 0, 'b': 1, 'c': 0}, expected_type=int), 1,
+ msg='fn should accept kwargs')
+ self.assertEqual(try_call(lambda: 1, expected_type=dict), None,
+ msg='int fn with no expected_type should give None')
+ self.assertEqual(try_call(lambda x: {}, total, args=(42, ), expected_type=int), 42,
+ msg='expect first int result with expected_type int')
+
+ def test_variadic(self):
+ self.assertEqual(variadic(None), (None, ))
+ self.assertEqual(variadic('spam'), ('spam', ))
+ self.assertEqual(variadic('spam', allowed_types=dict), 'spam')
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore')
+ self.assertEqual(variadic('spam', allowed_types=[dict]), 'spam')
+
+ def test_http_header_dict(self):
+ headers = HTTPHeaderDict()
+ headers['ytdl-test'] = b'0'
+ self.assertEqual(list(headers.items()), [('Ytdl-Test', '0')])
+ headers['ytdl-test'] = 1
+ self.assertEqual(list(headers.items()), [('Ytdl-Test', '1')])
+ headers['Ytdl-test'] = '2'
+ self.assertEqual(list(headers.items()), [('Ytdl-Test', '2')])
+ self.assertTrue('ytDl-Test' in headers)
+ self.assertEqual(str(headers), str(dict(headers)))
+ self.assertEqual(repr(headers), str(dict(headers)))
+
+ headers.update({'X-dlp': 'data'})
+ self.assertEqual(set(headers.items()), {('Ytdl-Test', '2'), ('X-Dlp', 'data')})
+ self.assertEqual(dict(headers), {'Ytdl-Test': '2', 'X-Dlp': 'data'})
+ self.assertEqual(len(headers), 2)
+ self.assertEqual(headers.copy(), headers)
+ headers2 = HTTPHeaderDict({'X-dlp': 'data3'}, **headers, **{'X-dlp': 'data2'})
+ self.assertEqual(set(headers2.items()), {('Ytdl-Test', '2'), ('X-Dlp', 'data2')})
+ self.assertEqual(len(headers2), 2)
+ headers2.clear()
+ self.assertEqual(len(headers2), 0)
+
+ # ensure we prefer latter headers
+ headers3 = HTTPHeaderDict({'Ytdl-TeSt': 1}, {'Ytdl-test': 2})
+ self.assertEqual(set(headers3.items()), {('Ytdl-Test', '2')})
+ del headers3['ytdl-tesT']
+ self.assertEqual(dict(headers3), {})
+
+ headers4 = HTTPHeaderDict({'ytdl-test': 'data;'})
+ self.assertEqual(set(headers4.items()), {('Ytdl-Test', 'data;')})
+
+ # common mistake: strip whitespace from values
+ # https://github.com/yt-dlp/yt-dlp/issues/8729
+ headers5 = HTTPHeaderDict({'ytdl-test': ' data; '})
+ self.assertEqual(set(headers5.items()), {('Ytdl-Test', 'data;')})
+
+ def test_extract_basic_auth(self):
+ assert extract_basic_auth('http://:foo.bar') == ('http://:foo.bar', None)
+ assert extract_basic_auth('http://foo.bar') == ('http://foo.bar', None)
+ assert extract_basic_auth('http://@foo.bar') == ('http://foo.bar', 'Basic Og==')
+ assert extract_basic_auth('http://:pass@foo.bar') == ('http://foo.bar', 'Basic OnBhc3M=')
+ assert extract_basic_auth('http://user:@foo.bar') == ('http://foo.bar', 'Basic dXNlcjo=')
+ assert extract_basic_auth('http://user:pass@foo.bar') == ('http://foo.bar', 'Basic dXNlcjpwYXNz')
+
+ @unittest.skipUnless(compat_os_name == 'nt', 'Only relevant on Windows')
+ def test_windows_escaping(self):
+ tests = [
+ 'test"&',
+ '%CMDCMDLINE:~-1%&',
+ 'a\nb',
+ '"',
+ '\\',
+ '!',
+ '^!',
+ 'a \\ b',
+ 'a \\" b',
+ 'a \\ b\\',
+ # We replace \r with \n
+ ('a\r\ra', 'a\n\na'),