import gzip
import hashlib
import hmac
-import imp
+import importlib.util
import io
import itertools
import json
import traceback
import xml.etree.ElementTree
import zlib
+import mimetypes
from .compat import (
compat_HTMLParseError,
compat_kwargs,
compat_os_name,
compat_parse_qs,
+ compat_shlex_split,
compat_shlex_quote,
compat_str,
compat_struct_pack,
def random_user_agent():
_USER_AGENT_TPL = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s Safari/537.36'
_CHROME_VERSIONS = (
- '74.0.3729.129',
- '76.0.3780.3',
- '76.0.3780.2',
- '74.0.3729.128',
- '76.0.3780.1',
- '76.0.3780.0',
- '75.0.3770.15',
- '74.0.3729.127',
- '74.0.3729.126',
- '76.0.3779.1',
- '76.0.3779.0',
- '75.0.3770.14',
- '74.0.3729.125',
- '76.0.3778.1',
- '76.0.3778.0',
- '75.0.3770.13',
- '74.0.3729.124',
- '74.0.3729.123',
- '73.0.3683.121',
- '76.0.3777.1',
- '76.0.3777.0',
- '75.0.3770.12',
- '74.0.3729.122',
- '76.0.3776.4',
- '75.0.3770.11',
- '74.0.3729.121',
- '76.0.3776.3',
- '76.0.3776.2',
- '73.0.3683.120',
- '74.0.3729.120',
- '74.0.3729.119',
- '74.0.3729.118',
- '76.0.3776.1',
- '76.0.3776.0',
- '76.0.3775.5',
- '75.0.3770.10',
- '74.0.3729.117',
- '76.0.3775.4',
- '76.0.3775.3',
- '74.0.3729.116',
- '75.0.3770.9',
- '76.0.3775.2',
- '76.0.3775.1',
- '76.0.3775.0',
- '75.0.3770.8',
- '74.0.3729.115',
- '74.0.3729.114',
- '76.0.3774.1',
- '76.0.3774.0',
- '75.0.3770.7',
- '74.0.3729.113',
- '74.0.3729.112',
- '74.0.3729.111',
- '76.0.3773.1',
- '76.0.3773.0',
- '75.0.3770.6',
- '74.0.3729.110',
- '74.0.3729.109',
- '76.0.3772.1',
- '76.0.3772.0',
- '75.0.3770.5',
- '74.0.3729.108',
- '74.0.3729.107',
- '76.0.3771.1',
- '76.0.3771.0',
- '75.0.3770.4',
- '74.0.3729.106',
- '74.0.3729.105',
- '75.0.3770.3',
- '74.0.3729.104',
- '74.0.3729.103',
- '74.0.3729.102',
- '75.0.3770.2',
- '74.0.3729.101',
- '75.0.3770.1',
- '75.0.3770.0',
- '74.0.3729.100',
- '75.0.3769.5',
- '75.0.3769.4',
- '74.0.3729.99',
- '75.0.3769.3',
- '75.0.3769.2',
- '75.0.3768.6',
- '74.0.3729.98',
- '75.0.3769.1',
- '75.0.3769.0',
- '74.0.3729.97',
- '73.0.3683.119',
- '73.0.3683.118',
- '74.0.3729.96',
- '75.0.3768.5',
- '75.0.3768.4',
- '75.0.3768.3',
- '75.0.3768.2',
- '74.0.3729.95',
- '74.0.3729.94',
- '75.0.3768.1',
- '75.0.3768.0',
- '74.0.3729.93',
- '74.0.3729.92',
- '73.0.3683.117',
- '74.0.3729.91',
- '75.0.3766.3',
- '74.0.3729.90',
- '75.0.3767.2',
- '75.0.3767.1',
- '75.0.3767.0',
- '74.0.3729.89',
- '73.0.3683.116',
- '75.0.3766.2',
- '74.0.3729.88',
- '75.0.3766.1',
- '75.0.3766.0',
- '74.0.3729.87',
- '73.0.3683.115',
- '74.0.3729.86',
- '75.0.3765.1',
- '75.0.3765.0',
- '74.0.3729.85',
- '73.0.3683.114',
- '74.0.3729.84',
- '75.0.3764.1',
- '75.0.3764.0',
- '74.0.3729.83',
- '73.0.3683.113',
- '75.0.3763.2',
- '75.0.3761.4',
- '74.0.3729.82',
- '75.0.3763.1',
- '75.0.3763.0',
- '74.0.3729.81',
- '73.0.3683.112',
- '75.0.3762.1',
- '75.0.3762.0',
- '74.0.3729.80',
- '75.0.3761.3',
- '74.0.3729.79',
- '73.0.3683.111',
- '75.0.3761.2',
- '74.0.3729.78',
- '74.0.3729.77',
- '75.0.3761.1',
- '75.0.3761.0',
- '73.0.3683.110',
- '74.0.3729.76',
- '74.0.3729.75',
- '75.0.3760.0',
- '74.0.3729.74',
- '75.0.3759.8',
- '75.0.3759.7',
- '75.0.3759.6',
- '74.0.3729.73',
- '75.0.3759.5',
- '74.0.3729.72',
- '73.0.3683.109',
- '75.0.3759.4',
- '75.0.3759.3',
- '74.0.3729.71',
- '75.0.3759.2',
- '74.0.3729.70',
- '73.0.3683.108',
- '74.0.3729.69',
- '75.0.3759.1',
- '75.0.3759.0',
- '74.0.3729.68',
- '73.0.3683.107',
- '74.0.3729.67',
- '75.0.3758.1',
- '75.0.3758.0',
- '74.0.3729.66',
- '73.0.3683.106',
- '74.0.3729.65',
- '75.0.3757.1',
- '75.0.3757.0',
- '74.0.3729.64',
- '73.0.3683.105',
- '74.0.3729.63',
- '75.0.3756.1',
- '75.0.3756.0',
- '74.0.3729.62',
- '73.0.3683.104',
- '75.0.3755.3',
- '75.0.3755.2',
- '73.0.3683.103',
- '75.0.3755.1',
- '75.0.3755.0',
- '74.0.3729.61',
- '73.0.3683.102',
- '74.0.3729.60',
- '75.0.3754.2',
- '74.0.3729.59',
- '75.0.3753.4',
- '74.0.3729.58',
- '75.0.3754.1',
- '75.0.3754.0',
- '74.0.3729.57',
- '73.0.3683.101',
- '75.0.3753.3',
- '75.0.3752.2',
- '75.0.3753.2',
- '74.0.3729.56',
- '75.0.3753.1',
- '75.0.3753.0',
- '74.0.3729.55',
- '73.0.3683.100',
- '74.0.3729.54',
- '75.0.3752.1',
- '75.0.3752.0',
- '74.0.3729.53',
- '73.0.3683.99',
- '74.0.3729.52',
- '75.0.3751.1',
- '75.0.3751.0',
- '74.0.3729.51',
- '73.0.3683.98',
- '74.0.3729.50',
- '75.0.3750.0',
- '74.0.3729.49',
- '74.0.3729.48',
- '74.0.3729.47',
- '75.0.3749.3',
- '74.0.3729.46',
- '73.0.3683.97',
- '75.0.3749.2',
- '74.0.3729.45',
- '75.0.3749.1',
- '75.0.3749.0',
- '74.0.3729.44',
- '73.0.3683.96',
- '74.0.3729.43',
- '74.0.3729.42',
- '75.0.3748.1',
- '75.0.3748.0',
- '74.0.3729.41',
- '75.0.3747.1',
- '73.0.3683.95',
- '75.0.3746.4',
- '74.0.3729.40',
- '74.0.3729.39',
- '75.0.3747.0',
- '75.0.3746.3',
- '75.0.3746.2',
- '74.0.3729.38',
- '75.0.3746.1',
- '75.0.3746.0',
- '74.0.3729.37',
- '73.0.3683.94',
- '75.0.3745.5',
- '75.0.3745.4',
- '75.0.3745.3',
- '75.0.3745.2',
- '74.0.3729.36',
- '75.0.3745.1',
- '75.0.3745.0',
- '75.0.3744.2',
- '74.0.3729.35',
- '73.0.3683.93',
- '74.0.3729.34',
- '75.0.3744.1',
- '75.0.3744.0',
- '74.0.3729.33',
- '73.0.3683.92',
- '74.0.3729.32',
- '74.0.3729.31',
- '73.0.3683.91',
- '75.0.3741.2',
- '75.0.3740.5',
- '74.0.3729.30',
- '75.0.3741.1',
- '75.0.3741.0',
- '74.0.3729.29',
- '75.0.3740.4',
- '73.0.3683.90',
- '74.0.3729.28',
- '75.0.3740.3',
- '73.0.3683.89',
- '75.0.3740.2',
- '74.0.3729.27',
- '75.0.3740.1',
- '75.0.3740.0',
- '74.0.3729.26',
- '73.0.3683.88',
- '73.0.3683.87',
- '74.0.3729.25',
- '75.0.3739.1',
- '75.0.3739.0',
- '73.0.3683.86',
- '74.0.3729.24',
- '73.0.3683.85',
- '75.0.3738.4',
- '75.0.3738.3',
- '75.0.3738.2',
- '75.0.3738.1',
- '75.0.3738.0',
- '74.0.3729.23',
- '73.0.3683.84',
- '74.0.3729.22',
- '74.0.3729.21',
- '75.0.3737.1',
- '75.0.3737.0',
- '74.0.3729.20',
- '73.0.3683.83',
- '74.0.3729.19',
- '75.0.3736.1',
- '75.0.3736.0',
- '74.0.3729.18',
- '73.0.3683.82',
- '74.0.3729.17',
- '75.0.3735.1',
- '75.0.3735.0',
- '74.0.3729.16',
- '73.0.3683.81',
- '75.0.3734.1',
- '75.0.3734.0',
- '74.0.3729.15',
- '73.0.3683.80',
- '74.0.3729.14',
- '75.0.3733.1',
- '75.0.3733.0',
- '75.0.3732.1',
- '74.0.3729.13',
- '74.0.3729.12',
- '73.0.3683.79',
- '74.0.3729.11',
- '75.0.3732.0',
- '74.0.3729.10',
- '73.0.3683.78',
- '74.0.3729.9',
- '74.0.3729.8',
- '74.0.3729.7',
- '75.0.3731.3',
- '75.0.3731.2',
- '75.0.3731.0',
- '74.0.3729.6',
- '73.0.3683.77',
- '73.0.3683.76',
- '75.0.3730.5',
- '75.0.3730.4',
- '73.0.3683.75',
- '74.0.3729.5',
- '73.0.3683.74',
- '75.0.3730.3',
- '75.0.3730.2',
- '74.0.3729.4',
- '73.0.3683.73',
- '73.0.3683.72',
- '75.0.3730.1',
- '75.0.3730.0',
- '74.0.3729.3',
- '73.0.3683.71',
- '74.0.3729.2',
- '73.0.3683.70',
- '74.0.3729.1',
- '74.0.3729.0',
- '74.0.3726.4',
- '73.0.3683.69',
- '74.0.3726.3',
- '74.0.3728.0',
- '74.0.3726.2',
- '73.0.3683.68',
- '74.0.3726.1',
- '74.0.3726.0',
- '74.0.3725.4',
- '73.0.3683.67',
- '73.0.3683.66',
- '74.0.3725.3',
- '74.0.3725.2',
- '74.0.3725.1',
- '74.0.3724.8',
- '74.0.3725.0',
- '73.0.3683.65',
- '74.0.3724.7',
- '74.0.3724.6',
- '74.0.3724.5',
- '74.0.3724.4',
- '74.0.3724.3',
- '74.0.3724.2',
- '74.0.3724.1',
- '74.0.3724.0',
- '73.0.3683.64',
- '74.0.3723.1',
- '74.0.3723.0',
- '73.0.3683.63',
- '74.0.3722.1',
- '74.0.3722.0',
- '73.0.3683.62',
- '74.0.3718.9',
- '74.0.3702.3',
- '74.0.3721.3',
- '74.0.3721.2',
- '74.0.3721.1',
- '74.0.3721.0',
- '74.0.3720.6',
- '73.0.3683.61',
- '72.0.3626.122',
- '73.0.3683.60',
- '74.0.3720.5',
- '72.0.3626.121',
- '74.0.3718.8',
- '74.0.3720.4',
- '74.0.3720.3',
- '74.0.3718.7',
- '74.0.3720.2',
- '74.0.3720.1',
- '74.0.3720.0',
- '74.0.3718.6',
- '74.0.3719.5',
- '73.0.3683.59',
- '74.0.3718.5',
- '74.0.3718.4',
- '74.0.3719.4',
- '74.0.3719.3',
- '74.0.3719.2',
- '74.0.3719.1',
- '73.0.3683.58',
- '74.0.3719.0',
- '73.0.3683.57',
- '73.0.3683.56',
- '74.0.3718.3',
- '73.0.3683.55',
- '74.0.3718.2',
- '74.0.3718.1',
- '74.0.3718.0',
- '73.0.3683.54',
- '74.0.3717.2',
- '73.0.3683.53',
- '74.0.3717.1',
- '74.0.3717.0',
- '73.0.3683.52',
- '74.0.3716.1',
- '74.0.3716.0',
- '73.0.3683.51',
- '74.0.3715.1',
- '74.0.3715.0',
- '73.0.3683.50',
- '74.0.3711.2',
- '74.0.3714.2',
- '74.0.3713.3',
- '74.0.3714.1',
- '74.0.3714.0',
- '73.0.3683.49',
- '74.0.3713.1',
- '74.0.3713.0',
- '72.0.3626.120',
- '73.0.3683.48',
- '74.0.3712.2',
- '74.0.3712.1',
- '74.0.3712.0',
- '73.0.3683.47',
- '72.0.3626.119',
- '73.0.3683.46',
- '74.0.3710.2',
- '72.0.3626.118',
- '74.0.3711.1',
- '74.0.3711.0',
- '73.0.3683.45',
- '72.0.3626.117',
- '74.0.3710.1',
- '74.0.3710.0',
- '73.0.3683.44',
- '72.0.3626.116',
- '74.0.3709.1',
- '74.0.3709.0',
- '74.0.3704.9',
- '73.0.3683.43',
- '72.0.3626.115',
- '74.0.3704.8',
- '74.0.3704.7',
- '74.0.3708.0',
- '74.0.3706.7',
- '74.0.3704.6',
- '73.0.3683.42',
- '72.0.3626.114',
- '74.0.3706.6',
- '72.0.3626.113',
- '74.0.3704.5',
- '74.0.3706.5',
- '74.0.3706.4',
- '74.0.3706.3',
- '74.0.3706.2',
- '74.0.3706.1',
- '74.0.3706.0',
- '73.0.3683.41',
- '72.0.3626.112',
- '74.0.3705.1',
- '74.0.3705.0',
- '73.0.3683.40',
- '72.0.3626.111',
- '73.0.3683.39',
- '74.0.3704.4',
- '73.0.3683.38',
- '74.0.3704.3',
- '74.0.3704.2',
- '74.0.3704.1',
- '74.0.3704.0',
- '73.0.3683.37',
- '72.0.3626.110',
- '72.0.3626.109',
- '74.0.3703.3',
- '74.0.3703.2',
- '73.0.3683.36',
- '74.0.3703.1',
- '74.0.3703.0',
- '73.0.3683.35',
- '72.0.3626.108',
- '74.0.3702.2',
- '74.0.3699.3',
- '74.0.3702.1',
- '74.0.3702.0',
- '73.0.3683.34',
- '72.0.3626.107',
- '73.0.3683.33',
- '74.0.3701.1',
- '74.0.3701.0',
- '73.0.3683.32',
- '73.0.3683.31',
- '72.0.3626.105',
- '74.0.3700.1',
- '74.0.3700.0',
- '73.0.3683.29',
- '72.0.3626.103',
- '74.0.3699.2',
- '74.0.3699.1',
- '74.0.3699.0',
- '73.0.3683.28',
- '72.0.3626.102',
- '73.0.3683.27',
- '73.0.3683.26',
- '74.0.3698.0',
- '74.0.3696.2',
- '72.0.3626.101',
- '73.0.3683.25',
- '74.0.3696.1',
- '74.0.3696.0',
- '74.0.3694.8',
- '72.0.3626.100',
- '74.0.3694.7',
- '74.0.3694.6',
- '74.0.3694.5',
- '74.0.3694.4',
- '72.0.3626.99',
- '72.0.3626.98',
- '74.0.3694.3',
- '73.0.3683.24',
- '72.0.3626.97',
- '72.0.3626.96',
- '72.0.3626.95',
- '73.0.3683.23',
- '72.0.3626.94',
- '73.0.3683.22',
- '73.0.3683.21',
- '72.0.3626.93',
- '74.0.3694.2',
- '72.0.3626.92',
- '74.0.3694.1',
- '74.0.3694.0',
- '74.0.3693.6',
- '73.0.3683.20',
- '72.0.3626.91',
- '74.0.3693.5',
- '74.0.3693.4',
- '74.0.3693.3',
- '74.0.3693.2',
- '73.0.3683.19',
- '74.0.3693.1',
- '74.0.3693.0',
- '73.0.3683.18',
- '72.0.3626.90',
- '74.0.3692.1',
- '74.0.3692.0',
- '73.0.3683.17',
- '72.0.3626.89',
- '74.0.3687.3',
- '74.0.3691.1',
- '74.0.3691.0',
- '73.0.3683.16',
- '72.0.3626.88',
- '72.0.3626.87',
- '73.0.3683.15',
- '74.0.3690.1',
- '74.0.3690.0',
- '73.0.3683.14',
- '72.0.3626.86',
- '73.0.3683.13',
- '73.0.3683.12',
- '74.0.3689.1',
- '74.0.3689.0',
- '73.0.3683.11',
- '72.0.3626.85',
- '73.0.3683.10',
- '72.0.3626.84',
- '73.0.3683.9',
- '74.0.3688.1',
- '74.0.3688.0',
- '73.0.3683.8',
- '72.0.3626.83',
- '74.0.3687.2',
- '74.0.3687.1',
- '74.0.3687.0',
- '73.0.3683.7',
- '72.0.3626.82',
- '74.0.3686.4',
- '72.0.3626.81',
- '74.0.3686.3',
- '74.0.3686.2',
- '74.0.3686.1',
- '74.0.3686.0',
- '73.0.3683.6',
- '72.0.3626.80',
- '74.0.3685.1',
- '74.0.3685.0',
- '73.0.3683.5',
- '72.0.3626.79',
- '74.0.3684.1',
- '74.0.3684.0',
- '73.0.3683.4',
- '72.0.3626.78',
- '72.0.3626.77',
- '73.0.3683.3',
- '73.0.3683.2',
- '72.0.3626.76',
- '73.0.3683.1',
- '73.0.3683.0',
- '72.0.3626.75',
- '71.0.3578.141',
- '73.0.3682.1',
- '73.0.3682.0',
- '72.0.3626.74',
- '71.0.3578.140',
- '73.0.3681.4',
- '73.0.3681.3',
- '73.0.3681.2',
- '73.0.3681.1',
- '73.0.3681.0',
- '72.0.3626.73',
- '71.0.3578.139',
- '72.0.3626.72',
- '72.0.3626.71',
- '73.0.3680.1',
- '73.0.3680.0',
- '72.0.3626.70',
- '71.0.3578.138',
- '73.0.3678.2',
- '73.0.3679.1',
- '73.0.3679.0',
- '72.0.3626.69',
- '71.0.3578.137',
- '73.0.3678.1',
- '73.0.3678.0',
- '71.0.3578.136',
- '73.0.3677.1',
- '73.0.3677.0',
- '72.0.3626.68',
- '72.0.3626.67',
- '71.0.3578.135',
- '73.0.3676.1',
- '73.0.3676.0',
- '73.0.3674.2',
- '72.0.3626.66',
- '71.0.3578.134',
- '73.0.3674.1',
- '73.0.3674.0',
- '72.0.3626.65',
- '71.0.3578.133',
- '73.0.3673.2',
- '73.0.3673.1',
- '73.0.3673.0',
- '72.0.3626.64',
- '71.0.3578.132',
- '72.0.3626.63',
- '72.0.3626.62',
- '72.0.3626.61',
- '72.0.3626.60',
- '73.0.3672.1',
- '73.0.3672.0',
- '72.0.3626.59',
- '71.0.3578.131',
- '73.0.3671.3',
- '73.0.3671.2',
- '73.0.3671.1',
- '73.0.3671.0',
- '72.0.3626.58',
- '71.0.3578.130',
- '73.0.3670.1',
- '73.0.3670.0',
- '72.0.3626.57',
- '71.0.3578.129',
- '73.0.3669.1',
- '73.0.3669.0',
- '72.0.3626.56',
- '71.0.3578.128',
- '73.0.3668.2',
- '73.0.3668.1',
- '73.0.3668.0',
- '72.0.3626.55',
- '71.0.3578.127',
- '73.0.3667.2',
- '73.0.3667.1',
- '73.0.3667.0',
- '72.0.3626.54',
- '71.0.3578.126',
- '73.0.3666.1',
- '73.0.3666.0',
- '72.0.3626.53',
- '71.0.3578.125',
- '73.0.3665.4',
- '73.0.3665.3',
- '72.0.3626.52',
- '73.0.3665.2',
- '73.0.3664.4',
- '73.0.3665.1',
- '73.0.3665.0',
- '72.0.3626.51',
- '71.0.3578.124',
- '72.0.3626.50',
- '73.0.3664.3',
- '73.0.3664.2',
- '73.0.3664.1',
- '73.0.3664.0',
- '73.0.3663.2',
- '72.0.3626.49',
- '71.0.3578.123',
- '73.0.3663.1',
- '73.0.3663.0',
- '72.0.3626.48',
- '71.0.3578.122',
- '73.0.3662.1',
- '73.0.3662.0',
- '72.0.3626.47',
- '71.0.3578.121',
- '73.0.3661.1',
- '72.0.3626.46',
- '73.0.3661.0',
- '72.0.3626.45',
- '71.0.3578.120',
- '73.0.3660.2',
- '73.0.3660.1',
- '73.0.3660.0',
- '72.0.3626.44',
- '71.0.3578.119',
- '73.0.3659.1',
- '73.0.3659.0',
- '72.0.3626.43',
- '71.0.3578.118',
- '73.0.3658.1',
- '73.0.3658.0',
- '72.0.3626.42',
- '71.0.3578.117',
- '73.0.3657.1',
- '73.0.3657.0',
- '72.0.3626.41',
- '71.0.3578.116',
- '73.0.3656.1',
- '73.0.3656.0',
- '72.0.3626.40',
- '71.0.3578.115',
- '73.0.3655.1',
- '73.0.3655.0',
- '72.0.3626.39',
- '71.0.3578.114',
- '73.0.3654.1',
- '73.0.3654.0',
- '72.0.3626.38',
- '71.0.3578.113',
- '73.0.3653.1',
- '73.0.3653.0',
- '72.0.3626.37',
- '71.0.3578.112',
- '73.0.3652.1',
- '73.0.3652.0',
- '72.0.3626.36',
- '71.0.3578.111',
- '73.0.3651.1',
- '73.0.3651.0',
- '72.0.3626.35',
- '71.0.3578.110',
- '73.0.3650.1',
- '73.0.3650.0',
- '72.0.3626.34',
- '71.0.3578.109',
- '73.0.3649.1',
- '73.0.3649.0',
- '72.0.3626.33',
- '71.0.3578.108',
- '73.0.3648.2',
- '73.0.3648.1',
- '73.0.3648.0',
- '72.0.3626.32',
- '71.0.3578.107',
- '73.0.3647.2',
- '73.0.3647.1',
- '73.0.3647.0',
- '72.0.3626.31',
- '71.0.3578.106',
- '73.0.3635.3',
- '73.0.3646.2',
- '73.0.3646.1',
- '73.0.3646.0',
- '72.0.3626.30',
- '71.0.3578.105',
- '72.0.3626.29',
- '73.0.3645.2',
- '73.0.3645.1',
- '73.0.3645.0',
- '72.0.3626.28',
- '71.0.3578.104',
- '72.0.3626.27',
- '72.0.3626.26',
- '72.0.3626.25',
- '72.0.3626.24',
- '73.0.3644.0',
- '73.0.3643.2',
- '72.0.3626.23',
- '71.0.3578.103',
- '73.0.3643.1',
- '73.0.3643.0',
- '72.0.3626.22',
- '71.0.3578.102',
- '73.0.3642.1',
- '73.0.3642.0',
- '72.0.3626.21',
- '71.0.3578.101',
- '73.0.3641.1',
- '73.0.3641.0',
- '72.0.3626.20',
- '71.0.3578.100',
- '72.0.3626.19',
- '73.0.3640.1',
- '73.0.3640.0',
- '72.0.3626.18',
- '73.0.3639.1',
- '71.0.3578.99',
- '73.0.3639.0',
- '72.0.3626.17',
- '73.0.3638.2',
- '72.0.3626.16',
- '73.0.3638.1',
- '73.0.3638.0',
- '72.0.3626.15',
- '71.0.3578.98',
- '73.0.3635.2',
- '71.0.3578.97',
- '73.0.3637.1',
- '73.0.3637.0',
- '72.0.3626.14',
- '71.0.3578.96',
- '71.0.3578.95',
- '72.0.3626.13',
- '71.0.3578.94',
- '73.0.3636.2',
- '71.0.3578.93',
- '73.0.3636.1',
- '73.0.3636.0',
- '72.0.3626.12',
- '71.0.3578.92',
- '73.0.3635.1',
- '73.0.3635.0',
- '72.0.3626.11',
- '71.0.3578.91',
- '73.0.3634.2',
- '73.0.3634.1',
- '73.0.3634.0',
- '72.0.3626.10',
- '71.0.3578.90',
- '71.0.3578.89',
- '73.0.3633.2',
- '73.0.3633.1',
- '73.0.3633.0',
- '72.0.3610.4',
- '72.0.3626.9',
- '71.0.3578.88',
- '73.0.3632.5',
- '73.0.3632.4',
- '73.0.3632.3',
- '73.0.3632.2',
- '73.0.3632.1',
- '73.0.3632.0',
- '72.0.3626.8',
- '71.0.3578.87',
- '73.0.3631.2',
- '73.0.3631.1',
- '73.0.3631.0',
- '72.0.3626.7',
- '71.0.3578.86',
- '72.0.3626.6',
- '73.0.3630.1',
- '73.0.3630.0',
- '72.0.3626.5',
- '71.0.3578.85',
- '72.0.3626.4',
- '73.0.3628.3',
- '73.0.3628.2',
- '73.0.3629.1',
- '73.0.3629.0',
- '72.0.3626.3',
- '71.0.3578.84',
- '73.0.3628.1',
- '73.0.3628.0',
- '71.0.3578.83',
- '73.0.3627.1',
- '73.0.3627.0',
- '72.0.3626.2',
- '71.0.3578.82',
- '71.0.3578.81',
- '71.0.3578.80',
- '72.0.3626.1',
- '72.0.3626.0',
- '71.0.3578.79',
- '70.0.3538.124',
- '71.0.3578.78',
- '72.0.3623.4',
- '72.0.3625.2',
- '72.0.3625.1',
- '72.0.3625.0',
- '71.0.3578.77',
- '70.0.3538.123',
- '72.0.3624.4',
- '72.0.3624.3',
- '72.0.3624.2',
- '71.0.3578.76',
- '72.0.3624.1',
- '72.0.3624.0',
- '72.0.3623.3',
- '71.0.3578.75',
- '70.0.3538.122',
- '71.0.3578.74',
- '72.0.3623.2',
- '72.0.3610.3',
- '72.0.3623.1',
- '72.0.3623.0',
- '72.0.3622.3',
- '72.0.3622.2',
- '71.0.3578.73',
- '70.0.3538.121',
- '72.0.3622.1',
- '72.0.3622.0',
- '71.0.3578.72',
- '70.0.3538.120',
- '72.0.3621.1',
- '72.0.3621.0',
- '71.0.3578.71',
- '70.0.3538.119',
- '72.0.3620.1',
- '72.0.3620.0',
- '71.0.3578.70',
- '70.0.3538.118',
- '71.0.3578.69',
- '72.0.3619.1',
- '72.0.3619.0',
- '71.0.3578.68',
- '70.0.3538.117',
- '71.0.3578.67',
- '72.0.3618.1',
- '72.0.3618.0',
- '71.0.3578.66',
- '70.0.3538.116',
- '72.0.3617.1',
- '72.0.3617.0',
- '71.0.3578.65',
- '70.0.3538.115',
- '72.0.3602.3',
- '71.0.3578.64',
- '72.0.3616.1',
- '72.0.3616.0',
- '71.0.3578.63',
- '70.0.3538.114',
- '71.0.3578.62',
- '72.0.3615.1',
- '72.0.3615.0',
- '71.0.3578.61',
- '70.0.3538.113',
- '72.0.3614.1',
- '72.0.3614.0',
- '71.0.3578.60',
- '70.0.3538.112',
- '72.0.3613.1',
- '72.0.3613.0',
- '71.0.3578.59',
- '70.0.3538.111',
- '72.0.3612.2',
- '72.0.3612.1',
- '72.0.3612.0',
- '70.0.3538.110',
- '71.0.3578.58',
- '70.0.3538.109',
- '72.0.3611.2',
- '72.0.3611.1',
- '72.0.3611.0',
- '71.0.3578.57',
- '70.0.3538.108',
- '72.0.3610.2',
- '71.0.3578.56',
- '71.0.3578.55',
- '72.0.3610.1',
- '72.0.3610.0',
- '71.0.3578.54',
- '70.0.3538.107',
- '71.0.3578.53',
- '72.0.3609.3',
- '71.0.3578.52',
- '72.0.3609.2',
- '71.0.3578.51',
- '72.0.3608.5',
- '72.0.3609.1',
- '72.0.3609.0',
- '71.0.3578.50',
- '70.0.3538.106',
- '72.0.3608.4',
- '72.0.3608.3',
- '72.0.3608.2',
- '71.0.3578.49',
- '72.0.3608.1',
- '72.0.3608.0',
- '70.0.3538.105',
- '71.0.3578.48',
- '72.0.3607.1',
- '72.0.3607.0',
- '71.0.3578.47',
- '70.0.3538.104',
- '72.0.3606.2',
- '72.0.3606.1',
- '72.0.3606.0',
- '71.0.3578.46',
- '70.0.3538.103',
- '70.0.3538.102',
- '72.0.3605.3',
- '72.0.3605.2',
- '72.0.3605.1',
- '72.0.3605.0',
- '71.0.3578.45',
- '70.0.3538.101',
- '71.0.3578.44',
- '71.0.3578.43',
- '70.0.3538.100',
- '70.0.3538.99',
- '71.0.3578.42',
- '72.0.3604.1',
- '72.0.3604.0',
- '71.0.3578.41',
- '70.0.3538.98',
- '71.0.3578.40',
- '72.0.3603.2',
- '72.0.3603.1',
- '72.0.3603.0',
- '71.0.3578.39',
- '70.0.3538.97',
- '72.0.3602.2',
- '71.0.3578.38',
- '71.0.3578.37',
- '72.0.3602.1',
- '72.0.3602.0',
- '71.0.3578.36',
- '70.0.3538.96',
- '72.0.3601.1',
- '72.0.3601.0',
- '71.0.3578.35',
- '70.0.3538.95',
- '72.0.3600.1',
- '72.0.3600.0',
- '71.0.3578.34',
- '70.0.3538.94',
- '72.0.3599.3',
- '72.0.3599.2',
- '72.0.3599.1',
- '72.0.3599.0',
- '71.0.3578.33',
- '70.0.3538.93',
- '72.0.3598.1',
- '72.0.3598.0',
- '71.0.3578.32',
- '70.0.3538.87',
- '72.0.3597.1',
- '72.0.3597.0',
- '72.0.3596.2',
- '71.0.3578.31',
- '70.0.3538.86',
- '71.0.3578.30',
- '71.0.3578.29',
- '72.0.3596.1',
- '72.0.3596.0',
- '71.0.3578.28',
- '70.0.3538.85',
- '72.0.3595.2',
- '72.0.3591.3',
- '72.0.3595.1',
- '72.0.3595.0',
- '71.0.3578.27',
- '70.0.3538.84',
- '72.0.3594.1',
- '72.0.3594.0',
- '71.0.3578.26',
- '70.0.3538.83',
- '72.0.3593.2',
- '72.0.3593.1',
- '72.0.3593.0',
- '71.0.3578.25',
- '70.0.3538.82',
- '72.0.3589.3',
- '72.0.3592.2',
- '72.0.3592.1',
- '72.0.3592.0',
- '71.0.3578.24',
- '72.0.3589.2',
- '70.0.3538.81',
- '70.0.3538.80',
- '72.0.3591.2',
- '72.0.3591.1',
- '72.0.3591.0',
- '71.0.3578.23',
- '70.0.3538.79',
- '71.0.3578.22',
- '72.0.3590.1',
- '72.0.3590.0',
- '71.0.3578.21',
- '70.0.3538.78',
- '70.0.3538.77',
- '72.0.3589.1',
- '72.0.3589.0',
- '71.0.3578.20',
- '70.0.3538.76',
- '71.0.3578.19',
- '70.0.3538.75',
- '72.0.3588.1',
- '72.0.3588.0',
- '71.0.3578.18',
- '70.0.3538.74',
- '72.0.3586.2',
- '72.0.3587.0',
- '71.0.3578.17',
- '70.0.3538.73',
- '72.0.3586.1',
- '72.0.3586.0',
- '71.0.3578.16',
- '70.0.3538.72',
- '72.0.3585.1',
- '72.0.3585.0',
- '71.0.3578.15',
- '70.0.3538.71',
- '71.0.3578.14',
- '72.0.3584.1',
- '72.0.3584.0',
- '71.0.3578.13',
- '70.0.3538.70',
- '72.0.3583.2',
- '71.0.3578.12',
- '72.0.3583.1',
- '72.0.3583.0',
- '71.0.3578.11',
- '70.0.3538.69',
- '71.0.3578.10',
- '72.0.3582.0',
- '72.0.3581.4',
- '71.0.3578.9',
- '70.0.3538.67',
- '72.0.3581.3',
- '72.0.3581.2',
- '72.0.3581.1',
- '72.0.3581.0',
- '71.0.3578.8',
- '70.0.3538.66',
- '72.0.3580.1',
- '72.0.3580.0',
- '71.0.3578.7',
- '70.0.3538.65',
- '71.0.3578.6',
- '72.0.3579.1',
- '72.0.3579.0',
- '71.0.3578.5',
- '70.0.3538.64',
- '71.0.3578.4',
- '71.0.3578.3',
- '71.0.3578.2',
- '71.0.3578.1',
- '71.0.3578.0',
- '70.0.3538.63',
- '69.0.3497.128',
- '70.0.3538.62',
- '70.0.3538.61',
- '70.0.3538.60',
- '70.0.3538.59',
- '71.0.3577.1',
- '71.0.3577.0',
- '70.0.3538.58',
- '69.0.3497.127',
- '71.0.3576.2',
- '71.0.3576.1',
- '71.0.3576.0',
- '70.0.3538.57',
- '70.0.3538.56',
- '71.0.3575.2',
- '70.0.3538.55',
- '69.0.3497.126',
- '70.0.3538.54',
- '71.0.3575.1',
- '71.0.3575.0',
- '71.0.3574.1',
- '71.0.3574.0',
- '70.0.3538.53',
- '69.0.3497.125',
- '70.0.3538.52',
- '71.0.3573.1',
- '71.0.3573.0',
- '70.0.3538.51',
- '69.0.3497.124',
- '71.0.3572.1',
- '71.0.3572.0',
- '70.0.3538.50',
- '69.0.3497.123',
- '71.0.3571.2',
- '70.0.3538.49',
- '69.0.3497.122',
- '71.0.3571.1',
- '71.0.3571.0',
- '70.0.3538.48',
- '69.0.3497.121',
- '71.0.3570.1',
- '71.0.3570.0',
- '70.0.3538.47',
- '69.0.3497.120',
- '71.0.3568.2',
- '71.0.3569.1',
- '71.0.3569.0',
- '70.0.3538.46',
- '69.0.3497.119',
- '70.0.3538.45',
- '71.0.3568.1',
- '71.0.3568.0',
- '70.0.3538.44',
- '69.0.3497.118',
- '70.0.3538.43',
- '70.0.3538.42',
- '71.0.3567.1',
- '71.0.3567.0',
- '70.0.3538.41',
- '69.0.3497.117',
- '71.0.3566.1',
- '71.0.3566.0',
- '70.0.3538.40',
- '69.0.3497.116',
- '71.0.3565.1',
- '71.0.3565.0',
- '70.0.3538.39',
- '69.0.3497.115',
- '71.0.3564.1',
- '71.0.3564.0',
- '70.0.3538.38',
- '69.0.3497.114',
- '71.0.3563.0',
- '71.0.3562.2',
- '70.0.3538.37',
- '69.0.3497.113',
- '70.0.3538.36',
- '70.0.3538.35',
- '71.0.3562.1',
- '71.0.3562.0',
- '70.0.3538.34',
- '69.0.3497.112',
- '70.0.3538.33',
- '71.0.3561.1',
- '71.0.3561.0',
- '70.0.3538.32',
- '69.0.3497.111',
- '71.0.3559.6',
- '71.0.3560.1',
- '71.0.3560.0',
- '71.0.3559.5',
- '71.0.3559.4',
- '70.0.3538.31',
- '69.0.3497.110',
- '71.0.3559.3',
- '70.0.3538.30',
- '69.0.3497.109',
- '71.0.3559.2',
- '71.0.3559.1',
- '71.0.3559.0',
- '70.0.3538.29',
- '69.0.3497.108',
- '71.0.3558.2',
- '71.0.3558.1',
- '71.0.3558.0',
- '70.0.3538.28',
- '69.0.3497.107',
- '71.0.3557.2',
- '71.0.3557.1',
- '71.0.3557.0',
- '70.0.3538.27',
- '69.0.3497.106',
- '71.0.3554.4',
- '70.0.3538.26',
- '71.0.3556.1',
- '71.0.3556.0',
- '70.0.3538.25',
- '71.0.3554.3',
- '69.0.3497.105',
- '71.0.3554.2',
- '70.0.3538.24',
- '69.0.3497.104',
- '71.0.3555.2',
- '70.0.3538.23',
- '71.0.3555.1',
- '71.0.3555.0',
- '70.0.3538.22',
- '69.0.3497.103',
- '71.0.3554.1',
- '71.0.3554.0',
- '70.0.3538.21',
- '69.0.3497.102',
- '71.0.3553.3',
- '70.0.3538.20',
- '69.0.3497.101',
- '71.0.3553.2',
- '69.0.3497.100',
- '71.0.3553.1',
- '71.0.3553.0',
- '70.0.3538.19',
- '69.0.3497.99',
- '69.0.3497.98',
- '69.0.3497.97',
- '71.0.3552.6',
- '71.0.3552.5',
- '71.0.3552.4',
- '71.0.3552.3',
- '71.0.3552.2',
- '71.0.3552.1',
- '71.0.3552.0',
- '70.0.3538.18',
- '69.0.3497.96',
- '71.0.3551.3',
- '71.0.3551.2',
- '71.0.3551.1',
- '71.0.3551.0',
- '70.0.3538.17',
- '69.0.3497.95',
- '71.0.3550.3',
- '71.0.3550.2',
- '71.0.3550.1',
- '71.0.3550.0',
- '70.0.3538.16',
- '69.0.3497.94',
- '71.0.3549.1',
- '71.0.3549.0',
- '70.0.3538.15',
- '69.0.3497.93',
- '69.0.3497.92',
- '71.0.3548.1',
- '71.0.3548.0',
- '70.0.3538.14',
- '69.0.3497.91',
- '71.0.3547.1',
- '71.0.3547.0',
- '70.0.3538.13',
- '69.0.3497.90',
- '71.0.3546.2',
- '69.0.3497.89',
- '71.0.3546.1',
- '71.0.3546.0',
- '70.0.3538.12',
- '69.0.3497.88',
- '71.0.3545.4',
- '71.0.3545.3',
- '71.0.3545.2',
- '71.0.3545.1',
- '71.0.3545.0',
- '70.0.3538.11',
- '69.0.3497.87',
- '71.0.3544.5',
- '71.0.3544.4',
- '71.0.3544.3',
- '71.0.3544.2',
- '71.0.3544.1',
- '71.0.3544.0',
- '69.0.3497.86',
- '70.0.3538.10',
- '69.0.3497.85',
- '70.0.3538.9',
- '69.0.3497.84',
- '71.0.3543.4',
- '70.0.3538.8',
- '71.0.3543.3',
- '71.0.3543.2',
- '71.0.3543.1',
- '71.0.3543.0',
- '70.0.3538.7',
- '69.0.3497.83',
- '71.0.3542.2',
- '71.0.3542.1',
- '71.0.3542.0',
- '70.0.3538.6',
- '69.0.3497.82',
- '69.0.3497.81',
- '71.0.3541.1',
- '71.0.3541.0',
- '70.0.3538.5',
- '69.0.3497.80',
- '71.0.3540.1',
- '71.0.3540.0',
- '70.0.3538.4',
- '69.0.3497.79',
- '70.0.3538.3',
- '71.0.3539.1',
- '71.0.3539.0',
- '69.0.3497.78',
- '68.0.3440.134',
- '69.0.3497.77',
- '70.0.3538.2',
- '70.0.3538.1',
- '70.0.3538.0',
- '69.0.3497.76',
- '68.0.3440.133',
- '69.0.3497.75',
- '70.0.3537.2',
- '70.0.3537.1',
- '70.0.3537.0',
- '69.0.3497.74',
- '68.0.3440.132',
- '70.0.3536.0',
- '70.0.3535.5',
- '70.0.3535.4',
- '70.0.3535.3',
- '69.0.3497.73',
- '68.0.3440.131',
- '70.0.3532.8',
- '70.0.3532.7',
- '69.0.3497.72',
- '69.0.3497.71',
- '70.0.3535.2',
- '70.0.3535.1',
- '70.0.3535.0',
- '69.0.3497.70',
- '68.0.3440.130',
- '69.0.3497.69',
- '68.0.3440.129',
- '70.0.3534.4',
- '70.0.3534.3',
- '70.0.3534.2',
- '70.0.3534.1',
- '70.0.3534.0',
- '69.0.3497.68',
- '68.0.3440.128',
- '70.0.3533.2',
- '70.0.3533.1',
- '70.0.3533.0',
- '69.0.3497.67',
- '68.0.3440.127',
- '70.0.3532.6',
- '70.0.3532.5',
- '70.0.3532.4',
- '69.0.3497.66',
- '68.0.3440.126',
- '70.0.3532.3',
- '70.0.3532.2',
- '70.0.3532.1',
- '69.0.3497.60',
- '69.0.3497.65',
- '69.0.3497.64',
- '70.0.3532.0',
- '70.0.3531.0',
- '70.0.3530.4',
- '70.0.3530.3',
- '70.0.3530.2',
- '69.0.3497.58',
- '68.0.3440.125',
- '69.0.3497.57',
- '69.0.3497.56',
- '69.0.3497.55',
- '69.0.3497.54',
- '70.0.3530.1',
- '70.0.3530.0',
- '69.0.3497.53',
- '68.0.3440.124',
- '69.0.3497.52',
- '70.0.3529.3',
- '70.0.3529.2',
- '70.0.3529.1',
- '70.0.3529.0',
- '69.0.3497.51',
- '70.0.3528.4',
- '68.0.3440.123',
- '70.0.3528.3',
- '70.0.3528.2',
- '70.0.3528.1',
- '70.0.3528.0',
- '69.0.3497.50',
- '68.0.3440.122',
- '70.0.3527.1',
- '70.0.3527.0',
- '69.0.3497.49',
- '68.0.3440.121',
- '70.0.3526.1',
- '70.0.3526.0',
- '68.0.3440.120',
- '69.0.3497.48',
- '69.0.3497.47',
- '68.0.3440.119',
- '68.0.3440.118',
- '70.0.3525.5',
- '70.0.3525.4',
- '70.0.3525.3',
- '68.0.3440.117',
- '69.0.3497.46',
- '70.0.3525.2',
- '70.0.3525.1',
- '70.0.3525.0',
- '69.0.3497.45',
- '68.0.3440.116',
- '70.0.3524.4',
- '70.0.3524.3',
- '69.0.3497.44',
- '70.0.3524.2',
- '70.0.3524.1',
- '70.0.3524.0',
- '70.0.3523.2',
- '69.0.3497.43',
- '68.0.3440.115',
- '70.0.3505.9',
- '69.0.3497.42',
- '70.0.3505.8',
- '70.0.3523.1',
- '70.0.3523.0',
- '69.0.3497.41',
- '68.0.3440.114',
- '70.0.3505.7',
- '69.0.3497.40',
- '70.0.3522.1',
- '70.0.3522.0',
- '70.0.3521.2',
- '69.0.3497.39',
- '68.0.3440.113',
- '70.0.3505.6',
- '70.0.3521.1',
- '70.0.3521.0',
- '69.0.3497.38',
- '68.0.3440.112',
- '70.0.3520.1',
- '70.0.3520.0',
- '69.0.3497.37',
- '68.0.3440.111',
- '70.0.3519.3',
- '70.0.3519.2',
- '70.0.3519.1',
- '70.0.3519.0',
- '69.0.3497.36',
- '68.0.3440.110',
- '70.0.3518.1',
- '70.0.3518.0',
- '69.0.3497.35',
- '69.0.3497.34',
- '68.0.3440.109',
- '70.0.3517.1',
- '70.0.3517.0',
- '69.0.3497.33',
- '68.0.3440.108',
- '69.0.3497.32',
- '70.0.3516.3',
- '70.0.3516.2',
- '70.0.3516.1',
- '70.0.3516.0',
- '69.0.3497.31',
- '68.0.3440.107',
- '70.0.3515.4',
- '68.0.3440.106',
- '70.0.3515.3',
- '70.0.3515.2',
- '70.0.3515.1',
- '70.0.3515.0',
- '69.0.3497.30',
- '68.0.3440.105',
- '68.0.3440.104',
- '70.0.3514.2',
- '70.0.3514.1',
- '70.0.3514.0',
- '69.0.3497.29',
- '68.0.3440.103',
- '70.0.3513.1',
- '70.0.3513.0',
- '69.0.3497.28',
+ '90.0.4430.212',
+ '90.0.4430.24',
+ '90.0.4430.70',
+ '90.0.4430.72',
+ '90.0.4430.85',
+ '90.0.4430.93',
+ '91.0.4472.101',
+ '91.0.4472.106',
+ '91.0.4472.114',
+ '91.0.4472.124',
+ '91.0.4472.164',
+ '91.0.4472.19',
+ '91.0.4472.77',
+ '92.0.4515.107',
+ '92.0.4515.115',
+ '92.0.4515.131',
+ '92.0.4515.159',
+ '92.0.4515.43',
+ '93.0.4556.0',
+ '93.0.4577.15',
+ '93.0.4577.63',
+ '93.0.4577.82',
+ '94.0.4606.41',
+ '94.0.4606.54',
+ '94.0.4606.61',
+ '94.0.4606.71',
+ '94.0.4606.81',
+ '94.0.4606.85',
+ '95.0.4638.17',
+ '95.0.4638.50',
+ '95.0.4638.54',
+ '95.0.4638.69',
+ '95.0.4638.74',
+ '96.0.4664.18',
+ '96.0.4664.45',
+ '96.0.4664.55',
+ '96.0.4664.93',
+ '97.0.4692.20',
)
return _USER_AGENT_TPL % random.choice(_CHROME_VERSIONS)
std_headers = {
'User-Agent': random_user_agent(),
- 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Encoding': 'gzip, deflate',
'Accept-Language': 'en-us,en;q=0.5',
+ 'Sec-Fetch-Mode': 'navigate',
}
'%Y/%m/%d %H:%M:%S',
'%Y%m%d%H%M',
'%Y%m%d%H%M%S',
+ '%Y%m%d',
'%Y-%m-%d %H:%M',
'%Y-%m-%d %H:%M:%S',
'%Y-%m-%d %H:%M:%S.%f',
try:
with tf:
- json.dump(obj, tf)
+ json.dump(obj, tf, ensure_ascii=False)
if sys.platform == 'win32':
# Need to remove existing file on Windows, else os.rename raises
# WindowsError or FileExistsError.
return get_element_by_attribute('id', id, html)
+def get_element_html_by_id(id, html):
+ """Return the html of the tag with the specified ID in the passed HTML document"""
+ return get_element_html_by_attribute('id', id, html)
+
+
def get_element_by_class(class_name, html):
"""Return the content of the first tag with the specified class in the passed HTML document"""
retval = get_elements_by_class(class_name, html)
return retval[0] if retval else None
+def get_element_html_by_class(class_name, html):
+ """Return the html of the first tag with the specified class in the passed HTML document"""
+ retval = get_elements_html_by_class(class_name, html)
+ return retval[0] if retval else None
+
+
def get_element_by_attribute(attribute, value, html, escape_value=True):
retval = get_elements_by_attribute(attribute, value, html, escape_value)
return retval[0] if retval else None
+def get_element_html_by_attribute(attribute, value, html, escape_value=True):
+ retval = get_elements_html_by_attribute(attribute, value, html, escape_value)
+ return retval[0] if retval else None
+
+
def get_elements_by_class(class_name, html):
"""Return the content of all tags with the specified class in the passed HTML document as a list"""
return get_elements_by_attribute(
html, escape_value=False)
-def get_elements_by_attribute(attribute, value, html, escape_value=True):
+def get_elements_html_by_class(class_name, html):
+ """Return the html of all tags with the specified class in the passed HTML document as a list"""
+ return get_elements_html_by_attribute(
+ 'class', r'[^\'"]*\b%s\b[^\'"]*' % re.escape(class_name),
+ html, escape_value=False)
+
+
+def get_elements_by_attribute(*args, **kwargs):
"""Return the content of the tag with the specified attribute in the passed HTML document"""
+ return [content for content, _ in get_elements_text_and_html_by_attribute(*args, **kwargs)]
+
+
+def get_elements_html_by_attribute(*args, **kwargs):
+ """Return the html of the tag with the specified attribute in the passed HTML document"""
+ return [whole for _, whole in get_elements_text_and_html_by_attribute(*args, **kwargs)]
+
+
+def get_elements_text_and_html_by_attribute(attribute, value, html, escape_value=True):
+ """
+ Return the text (content) and the html (whole) of the tag with the specified
+ attribute in the passed HTML document
+ """
+
+ value_quote_optional = '' if re.match(r'''[\s"'`=<>]''', value) else '?'
value = re.escape(value) if escape_value else value
- retlist = []
- for m in re.finditer(r'''(?xs)
- <([a-zA-Z0-9:._-]+)
- (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]*|="[^"]*"|='[^']*'|))*?
- \s+%s=['"]?%s['"]?
- (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]*|="[^"]*"|='[^']*'|))*?
- \s*>
- (?P<content>.*?)
- </\1>
- ''' % (re.escape(attribute), value), html):
- res = m.group('content')
+ partial_element_re = r'''(?x)
+ <(?P<tag>[a-zA-Z0-9:._-]+)
+ (?:\s(?:[^>"']|"[^"]*"|'[^']*')*)?
+ \s%(attribute)s\s*=\s*(?P<_q>['"]%(vqo)s)(?-x:%(value)s)(?P=_q)
+ ''' % {'attribute': re.escape(attribute), 'value': value, 'vqo': value_quote_optional}
+
+ for m in re.finditer(partial_element_re, html):
+ content, whole = get_element_text_and_html_by_tag(m.group('tag'), html[m.start():])
+
+ yield (
+ unescapeHTML(re.sub(r'^(?P<q>["\'])(?P<content>.*)(?P=q)$', r'\g<content>', content, flags=re.DOTALL)),
+ whole
+ )
+
+
+class HTMLBreakOnClosingTagParser(compat_HTMLParser):
+ """
+ HTML parser which raises HTMLBreakOnClosingTagException upon reaching the
+ closing tag for the first opening tag it has encountered, and can be used
+ as a context manager
+ """
+
+ class HTMLBreakOnClosingTagException(Exception):
+ pass
- if res.startswith('"') or res.startswith("'"):
- res = res[1:-1]
+ def __init__(self):
+ self.tagstack = collections.deque()
+ compat_HTMLParser.__init__(self)
- retlist.append(unescapeHTML(res))
+ def __enter__(self):
+ return self
- return retlist
+ def __exit__(self, *_):
+ self.close()
+
+ def close(self):
+ # handle_endtag does not return upon raising HTMLBreakOnClosingTagException,
+ # so data remains buffered; we no longer have any interest in it, thus
+ # override this method to discard it
+ pass
+
+ def handle_starttag(self, tag, _):
+ self.tagstack.append(tag)
+
+ def handle_endtag(self, tag):
+ if not self.tagstack:
+ raise compat_HTMLParseError('no tags in the stack')
+ while self.tagstack:
+ inner_tag = self.tagstack.pop()
+ if inner_tag == tag:
+ break
+ else:
+ raise compat_HTMLParseError(f'matching opening tag for closing {tag} tag not found')
+ if not self.tagstack:
+ raise self.HTMLBreakOnClosingTagException()
+
+
+def get_element_text_and_html_by_tag(tag, html):
+ """
+ For the first element with the specified tag in the passed HTML document
+ return its' content (text) and the whole element (html)
+ """
+ def find_or_raise(haystack, needle, exc):
+ try:
+ return haystack.index(needle)
+ except ValueError:
+ raise exc
+ closing_tag = f'</{tag}>'
+ whole_start = find_or_raise(
+ html, f'<{tag}', compat_HTMLParseError(f'opening {tag} tag not found'))
+ content_start = find_or_raise(
+ html[whole_start:], '>', compat_HTMLParseError(f'malformed opening {tag} tag'))
+ content_start += whole_start + 1
+ with HTMLBreakOnClosingTagParser() as parser:
+ parser.feed(html[whole_start:content_start])
+ if not parser.tagstack or parser.tagstack[0] != tag:
+ raise compat_HTMLParseError(f'parser did not match opening {tag} tag')
+ offset = content_start
+ while offset < len(html):
+ next_closing_tag_start = find_or_raise(
+ html[offset:], closing_tag,
+ compat_HTMLParseError(f'closing {tag} tag not found'))
+ next_closing_tag_end = next_closing_tag_start + len(closing_tag)
+ try:
+ parser.feed(html[offset:offset + next_closing_tag_end])
+ offset += next_closing_tag_end
+ except HTMLBreakOnClosingTagParser.HTMLBreakOnClosingTagException:
+ return html[content_start:offset + next_closing_tag_start], \
+ html[whole_start:offset + next_closing_tag_end]
+ raise compat_HTMLParseError('unexpected end of html')
class HTMLAttributeParser(compat_HTMLParser):
self.attrs = dict(attrs)
+class HTMLListAttrsParser(compat_HTMLParser):
+ """HTML parser to gather the attributes for the elements of a list"""
+
+ def __init__(self):
+ compat_HTMLParser.__init__(self)
+ self.items = []
+ self._level = 0
+
+ def handle_starttag(self, tag, attrs):
+ if tag == 'li' and self._level == 0:
+ self.items.append(dict(attrs))
+ self._level += 1
+
+ def handle_endtag(self, tag):
+ self._level -= 1
+
+
def extract_attributes(html_element):
"""Given a string for an HTML element such as
<el
return parser.attrs
+def parse_list(webpage):
+ """Given a string for an series of HTML <li> elements,
+ return a dictionary of their attributes"""
+ parser = HTMLListAttrsParser()
+ parser.feed(webpage)
+ parser.close()
+ return parser.items
+
+
def clean_html(html):
"""Clean an HTML snippet into a readable string"""
if html is None: # Convenience for sanitizing descriptions etc.
return html
- # Newline vs <br />
- html = html.replace('\n', ' ')
- html = re.sub(r'(?u)\s*<\s*br\s*/?\s*>\s*', '\n', html)
- html = re.sub(r'(?u)<\s*/\s*p\s*>\s*<\s*p[^>]*>', '\n', html)
+ html = re.sub(r'\s+', ' ', html)
+ html = re.sub(r'(?u)\s?<\s?br\s?/?\s?>\s?', '\n', html)
+ html = re.sub(r'(?u)<\s?/\s?p\s?>\s?<\s?p[^>]*>', '\n', html)
# Strip html tags
html = re.sub('<.*?>', '', html)
# Replace html entities
raise
+class Popen(subprocess.Popen):
+ if sys.platform == 'win32':
+ _startupinfo = subprocess.STARTUPINFO()
+ _startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
+ else:
+ _startupinfo = None
+
+ def __init__(self, *args, **kwargs):
+ super(Popen, self).__init__(*args, **kwargs, startupinfo=self._startupinfo)
+
+ def communicate_or_kill(self, *args, **kwargs):
+ return process_communicate_or_kill(self, *args, **kwargs)
+
+
def get_subprocess_encoding():
if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
# For subprocess calls, encode with locale encoding
return optval
+_timetuple = collections.namedtuple('Time', ('hours', 'minutes', 'seconds', 'milliseconds'))
+
+
+def timetuple_from_msec(msec):
+ secs, msec = divmod(msec, 1000)
+ mins, secs = divmod(secs, 60)
+ hrs, mins = divmod(mins, 60)
+ return _timetuple(hrs, mins, secs, msec)
+
+
def formatSeconds(secs, delim=':', msec=False):
- if secs > 3600:
- ret = '%d%s%02d%s%02d' % (secs // 3600, delim, (secs % 3600) // 60, delim, secs % 60)
- elif secs > 60:
- ret = '%d%s%02d' % (secs // 60, delim, secs % 60)
+ time = timetuple_from_msec(secs * 1000)
+ if time.hours:
+ ret = '%d%s%02d%s%02d' % (time.hours, delim, time.minutes, delim, time.seconds)
+ elif time.minutes:
+ ret = '%d%s%02d' % (time.minutes, delim, time.seconds)
else:
- ret = '%d' % secs
- return '%s.%03d' % (ret, secs % 1) if msec else ret
+ ret = '%d' % time.seconds
+ return '%s.%03d' % (ret, time.milliseconds) if msec else ret
def _ssl_load_windows_store_certs(ssl_context, storename):
opts_check_certificate = not params.get('nocheckcertificate')
context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
context.check_hostname = opts_check_certificate
+ if params.get('legacyserverconnect'):
+ context.options |= 4 # SSL_OP_LEGACY_SERVER_CONNECT
context.verify_mode = ssl.CERT_REQUIRED if opts_check_certificate else ssl.CERT_NONE
if opts_check_certificate:
- # Work around the issue in load_default_certs when there are bad certificates. See:
- # https://github.com/yt-dlp/yt-dlp/issues/1060,
- # https://bugs.python.org/issue35665, https://bugs.python.org/issue4531
- if sys.platform == 'win32':
- for storename in ('CA', 'ROOT'):
- _ssl_load_windows_store_certs(context, storename)
- context.set_default_verify_paths()
+ try:
+ context.load_default_certs()
+ # Work around the issue in load_default_certs when there are bad certificates. See:
+ # https://github.com/yt-dlp/yt-dlp/issues/1060,
+ # https://bugs.python.org/issue35665, https://bugs.python.org/issue45312
+ except ssl.SSLError:
+ # enum_certificates is not present in mingw python. See https://github.com/yt-dlp/yt-dlp/issues/1151
+ if sys.platform == 'win32' and hasattr(ssl, 'enum_certificates'):
+ # Create a new context to discard any certificates that were already loaded
+ context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+ context.check_hostname, context.verify_mode = True, ssl.CERT_REQUIRED
+ for storename in ('CA', 'ROOT'):
+ _ssl_load_windows_store_certs(context, storename)
+ context.set_default_verify_paths()
return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
def bug_reports_message(before=';'):
- if ytdl_is_updateable():
- update_cmd = 'type yt-dlp -U to update'
- else:
- update_cmd = 'see https://github.com/yt-dlp/yt-dlp on how to update'
- msg = 'please report this issue on https://github.com/yt-dlp/yt-dlp .'
- msg += ' Make sure you are using the latest version; %s.' % update_cmd
- msg += ' Be sure to call yt-dlp with the --verbose flag and include its complete output.'
+ msg = ('please report this issue on https://github.com/yt-dlp/yt-dlp , '
+ 'filling out the "Broken site" issue template properly. '
+ 'Confirm you are on the latest version using -U')
before = before.rstrip()
if not before or before.endswith(('.', '!', '?')):
class YoutubeDLError(Exception):
"""Base exception for YoutubeDL errors."""
- pass
+ msg = None
+
+ def __init__(self, msg=None):
+ if msg is not None:
+ self.msg = msg
+ elif self.msg is None:
+ self.msg = type(self).__name__
+ super().__init__(self.msg)
network_exceptions = [compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error]
geographic location due to geographic restrictions imposed by a website.
"""
- def __init__(self, msg, countries=None):
- super(GeoRestrictedError, self).__init__(msg, expected=True)
- self.msg = msg
+ def __init__(self, msg, countries=None, **kwargs):
+ kwargs['expected'] = True
+ super(GeoRestrictedError, self).__init__(msg, **kwargs)
self.countries = countries
This exception will be thrown by YoutubeDL when a requested entry
is not found in the playlist info_dict
"""
- pass
+ msg = 'Entry not found in info'
class SameFileError(YoutubeDLError):
This exception will be thrown by FileDownloader objects if they detect
multiple files would have to be downloaded to the same file on disk.
"""
- pass
+ msg = 'Fixed output name but more than one file to download'
+
+ def __init__(self, filename=None):
+ if filename is not None:
+ self.msg += f': {filename}'
+ super().__init__(self.msg)
class PostProcessingError(YoutubeDLError):
indicate an error in the postprocessing task.
"""
- def __init__(self, msg):
- super(PostProcessingError, self).__init__(msg)
- self.msg = msg
+class DownloadCancelled(YoutubeDLError):
+ """ Exception raised when the download queue should be interrupted """
+ msg = 'The download was cancelled'
-class ExistingVideoReached(YoutubeDLError):
- """ --max-downloads limit has been reached. """
- pass
+
+class ExistingVideoReached(DownloadCancelled):
+ """ --break-on-existing triggered """
+ msg = 'Encountered a video that is already in the archive, stopping due to --break-on-existing'
-class RejectedVideoReached(YoutubeDLError):
+class RejectedVideoReached(DownloadCancelled):
+ """ --break-on-reject triggered """
+ msg = 'Encountered a video that did not match filter, stopping due to --break-on-reject'
+
+
+class MaxDownloadsReached(DownloadCancelled):
""" --max-downloads limit has been reached. """
- pass
+ msg = 'Maximum number of downloads reached, stopping due to --max-downloads'
-class ThrottledDownload(YoutubeDLError):
- """ Download speed below --throttled-rate. """
- pass
+class ReExtractInfo(YoutubeDLError):
+ """ Video info needs to be re-extracted. """
+ def __init__(self, msg, expected=False):
+ super().__init__(msg)
+ self.expected = expected
-class MaxDownloadsReached(YoutubeDLError):
- """ --max-downloads limit has been reached. """
- pass
+
+class ThrottledDownload(ReExtractInfo):
+ """ Download speed below --throttled-rate. """
+ msg = 'The download speed is below throttle limit'
+
+ def __init__(self):
+ super().__init__(self.msg, expected=False)
class UnavailableVideoError(YoutubeDLError):
This exception will be thrown when a video is requested
in a format that is not available for that video.
"""
- pass
+ msg = 'Unable to download video'
+
+ def __init__(self, err=None):
+ if err is not None:
+ self.msg += f': {err}'
+ super().__init__(self.msg)
class ContentTooShortError(YoutubeDLError):
if precision == 'auto':
auto_precision = True
precision = 'microsecond'
- today = datetime_round(datetime.datetime.now(), precision)
+ today = datetime_round(datetime.datetime.utcnow(), precision)
if date_str in ('now', 'today'):
return today
if date_str == 'yesterday':
False if it has yet to be written out."""
# Adapted from http://stackoverflow.com/a/3259271/35070
- import ctypes
import ctypes.wintypes
WIN_OUTPUT_IDS = {
return url, data
+def format_decimal_suffix(num, fmt='%d%s', *, factor=1000):
+ """ Formats numbers with decimal sufixes like K, M, etc """
+ num, factor = float_or_none(num), float(factor)
+ if num is None:
+ return None
+ exponent = 0 if num == 0 else int(math.log(num, factor))
+ suffix = ['', *'kMGTPEZY'][exponent]
+ if factor == 1024:
+ suffix = {'k': 'Ki', '': ''}.get(suffix, f'{suffix}i')
+ converted = num / (factor ** exponent)
+ return fmt % (converted, suffix)
+
+
def format_bytes(bytes):
- if bytes is None:
- return 'N/A'
- if type(bytes) is str:
- bytes = float(bytes)
- if bytes == 0.0:
- exponent = 0
- else:
- exponent = int(math.log(bytes, 1024.0))
- suffix = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'][exponent]
- converted = float(bytes) / float(1024 ** exponent)
- return '%.2f%s' % (converted, suffix)
+ return format_decimal_suffix(bytes, '%.2f%sB', factor=1024) or 'N/A'
def lookup_unit_table(unit_table, s):
if s is None:
return None
- s = s.strip()
+ s = re.sub(r'^[^\d]+\s', '', s).strip()
if re.match(r'^[\d,.]+$', s):
return str_to_int(s)
'M': 1000 ** 2,
'kk': 1000 ** 2,
'KK': 1000 ** 2,
+ 'b': 1000 ** 3,
+ 'B': 1000 ** 3,
}
- return lookup_unit_table(_UNIT_TABLE, s)
+ ret = lookup_unit_table(_UNIT_TABLE, s)
+ if ret is not None:
+ return ret
+
+ mobj = re.match(r'([\d,.]+)(?:$|\s)', s)
+ if mobj:
+ return str_to_int(mobj.group(1))
def parse_resolution(s):
if s is None:
return {}
- mobj = re.search(r'\b(?P<w>\d+)\s*[xX×]\s*(?P<h>\d+)\b', s)
+ mobj = re.search(r'(?<![a-zA-Z0-9])(?P<w>\d+)\s*[xX×,]\s*(?P<h>\d+)(?![a-zA-Z0-9])', s)
if mobj:
return {
'width': int(mobj.group('w')),
'height': int(mobj.group('h')),
}
- mobj = re.search(r'\b(\d+)[pPiI]\b', s)
+ mobj = re.search(r'(?<![a-zA-Z0-9])(\d+)[pPiI](?![a-zA-Z0-9])', s)
if mobj:
return {'height': int(mobj.group(1))}
def int_or_none(v, scale=1, default=None, get_attr=None, invscale=1):
- if get_attr:
- if v is not None:
- v = getattr(v, get_attr, None)
- if v == '':
- v = None
- if v is None:
- return default
+ if get_attr and v is not None:
+ v = getattr(v, get_attr, None)
try:
return int(v) * invscale // scale
- except (ValueError, TypeError):
+ except (ValueError, TypeError, OverflowError):
return default
def parse_duration(s):
if not isinstance(s, compat_basestring):
return None
-
s = s.strip()
+ if not s:
+ return None
days, hours, mins, secs, ms = [None] * 5
- m = re.match(r'(?:(?:(?:(?P<days>[0-9]+):)?(?P<hours>[0-9]+):)?(?P<mins>[0-9]+):)?(?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?Z?$', s)
+ m = re.match(r'''(?x)
+ (?P<before_secs>
+ (?:(?:(?P<days>[0-9]+):)?(?P<hours>[0-9]+):)?(?P<mins>[0-9]+):)?
+ (?P<secs>(?(before_secs)[0-9]{1,2}|[0-9]+))
+ (?P<ms>[.:][0-9]+)?Z?$
+ ''', s)
if m:
- days, hours, mins, secs, ms = m.groups()
+ days, hours, mins, secs, ms = m.group('days', 'hours', 'mins', 'secs', 'ms')
else:
m = re.match(
r'''(?ix)(?:P?
if days:
duration += float(days) * 24 * 60 * 60
if ms:
- duration += float(ms)
+ duration += float(ms.replace(':', '.'))
return duration
""" Checks if the given binary is installed somewhere in PATH, and returns its name.
args can be a list of arguments for a short output (like -version) """
try:
- process_communicate_or_kill(subprocess.Popen(
- [exe] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE))
+ Popen([exe] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate_or_kill()
except OSError:
return False
return exe
-def get_exe_version(exe, args=['--version'],
- version_re=None, unrecognized='present'):
- """ Returns the version of the specified executable,
- or False if the executable is not present """
+def _get_exe_version_output(exe, args):
try:
# STDIN should be redirected too. On UNIX-like systems, ffmpeg triggers
# SIGTTOU if yt-dlp is run in the background.
# See https://github.com/ytdl-org/youtube-dl/issues/955#issuecomment-209789656
- out, _ = process_communicate_or_kill(subprocess.Popen(
- [encodeArgument(exe)] + args,
- stdin=subprocess.PIPE,
- stdout=subprocess.PIPE, stderr=subprocess.STDOUT))
+ out, _ = Popen(
+ [encodeArgument(exe)] + args, stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate_or_kill()
except OSError:
return False
if isinstance(out, bytes): # Python 2.x
out = out.decode('ascii', 'ignore')
- return detect_exe_version(out, version_re, unrecognized)
+ return out
def detect_exe_version(output, version_re=None, unrecognized='present'):
return unrecognized
+def get_exe_version(exe, args=['--version'],
+ version_re=None, unrecognized='present'):
+ """ Returns the version of the specified executable,
+ or False if the executable is not present """
+ out = _get_exe_version_output(exe, args)
+ return detect_exe_version(out, version_re, unrecognized) if out else False
+
+
class LazyList(collections.abc.Sequence):
''' Lazy immutable list from an iterable
Note that slices of a LazyList are lists and not LazyList'''
class IndexError(IndexError):
pass
- def __init__(self, iterable):
+ def __init__(self, iterable, *, reverse=False, _cache=None):
self.__iterable = iter(iterable)
- self.__cache = []
- self.__reversed = False
+ self.__cache = [] if _cache is None else _cache
+ self.__reversed = reverse
def __iter__(self):
if self.__reversed:
def __exhaust(self):
self.__cache.extend(self.__iterable)
+ # Discard the emptied iterable to make it pickle-able
+ self.__iterable = []
return self.__cache
def exhaust(self):
self.__exhaust()
return len(self.__cache)
- def reverse(self):
- self.__reversed = not self.__reversed
- return self
+ def __reversed__(self):
+ return type(self)(self.__iterable, reverse=not self.__reversed, _cache=self.__cache)
+
+ def __copy__(self):
+ return type(self)(self.__iterable, reverse=self.__reversed, _cache=self.__cache)
def __repr__(self):
# repr and str should mimic a list. So we exhaust the iterable
class PagedList:
+
+ class IndexError(IndexError):
+ pass
+
def __len__(self):
# This is only useful for tests
return len(self.getslice())
self._cache = {}
def getpage(self, pagenum):
- page_results = self._cache.get(pagenum) or list(self._pagefunc(pagenum))
+ page_results = self._cache.get(pagenum)
+ if page_results is None:
+ page_results = list(self._pagefunc(pagenum))
if self._use_cache:
self._cache[pagenum] = page_results
return page_results
if not isinstance(idx, int) or idx < 0:
raise TypeError('indices must be non-negative integers')
entries = self.getslice(idx, idx + 1)
- return entries[0] if entries else None
+ if not entries:
+ raise self.IndexError()
+ return entries[0]
class OnDemandPagedList(PagedList):
def _getslice(self, start, end):
start_page = start // self._pagesize
- end_page = (
- self._pagecount if end is None else (end // self._pagesize + 1))
+ end_page = self._pagecount if end is None else min(self._pagecount, end // self._pagesize + 1)
skip_elems = start - start_page * self._pagesize
only_more = None if end is None else end - start
for pagenum in range(start_page, end_page):
return q
+POSTPROCESS_WHEN = {'pre_process', 'before_dl', 'after_move', 'post_process', 'after_video', 'playlist'}
+
+
DEFAULT_OUTTMPL = {
'default': '%(title)s [%(id)s].%(ext)s',
'chapter': '%(title)s - %(section_number)03d %(section_title)s [%(id)s].%(ext)s',
'description': 'description',
'annotation': 'annotations.xml',
'infojson': 'info.json',
+ 'link': None,
+ 'pl_video': None,
'pl_thumbnail': None,
'pl_description': 'description',
'pl_infojson': 'info.json',
def ytdl_is_updateable():
""" Returns if yt-dlp can be updated with -U """
- return False
- from zipimport import zipimporter
+ from .update import is_non_updateable
- return isinstance(globals().get('__loader__'), zipimporter) or hasattr(sys, 'frozen')
+ return not is_non_updateable()
def args_to_str(args):
if mt is None:
return None
- ext = {
+ mt, _, params = mt.partition(';')
+ mt = mt.strip()
+
+ FULL_MAP = {
'audio/mp4': 'm4a',
# Per RFC 3003, audio/mpeg can be .mp1, .mp2 or .mp3. Here use .mp3 as
# it's the most popular one
'audio/mpeg': 'mp3',
'audio/x-wav': 'wav',
- }.get(mt)
+ 'audio/wav': 'wav',
+ 'audio/wave': 'wav',
+ }
+
+ ext = FULL_MAP.get(mt)
if ext is not None:
return ext
- _, _, res = mt.rpartition('/')
- res = res.split(';')[0].strip().lower()
-
- return {
+ SUBTYPE_MAP = {
'3gpp': '3gp',
'smptett+xml': 'tt',
'ttaf+xml': 'dfxp',
'quicktime': 'mov',
'mp2t': 'ts',
'x-wav': 'wav',
- }.get(res, res)
+ 'filmstrip+json': 'fs',
+ 'svg+xml': 'svg',
+ }
+
+ _, _, subtype = mt.rpartition('/')
+ ext = SUBTYPE_MAP.get(subtype.lower())
+ if ext is not None:
+ return ext
+
+ SUFFIX_MAP = {
+ 'json': 'json',
+ 'xml': 'xml',
+ 'zip': 'zip',
+ 'gzip': 'gz',
+ }
+
+ _, _, suffix = subtype.partition('+')
+ ext = SUFFIX_MAP.get(suffix)
+ if ext is not None:
+ return ext
+
+ return subtype.replace('+', '.')
+
+
+def ext2mimetype(ext_or_url):
+ if not ext_or_url:
+ return None
+ if '.' not in ext_or_url:
+ ext_or_url = f'file.{ext_or_url}'
+ return mimetypes.guess_type(ext_or_url)[0]
def parse_codecs(codecs_str):
return {}
split_codecs = list(filter(None, map(
str.strip, codecs_str.strip().strip(',').split(','))))
- vcodec, acodec = None, None
+ vcodec, acodec, tcodec, hdr = None, None, None, None
for full_codec in split_codecs:
- codec = full_codec.split('.')[0]
- if codec in ('avc1', 'avc2', 'avc3', 'avc4', 'vp9', 'vp8', 'hev1', 'hev2', 'h263', 'h264', 'mp4v', 'hvc1', 'av01', 'theora'):
+ parts = full_codec.split('.')
+ codec = parts[0].replace('0', '')
+ if codec in ('avc1', 'avc2', 'avc3', 'avc4', 'vp9', 'vp8', 'hev1', 'hev2',
+ 'h263', 'h264', 'mp4v', 'hvc1', 'av1', 'theora', 'dvh1', 'dvhe'):
if not vcodec:
- vcodec = full_codec
- elif codec in ('mp4a', 'opus', 'vorbis', 'mp3', 'aac', 'ac-3', 'ec-3', 'eac3', 'dtsc', 'dtse', 'dtsh', 'dtsl'):
+ vcodec = '.'.join(parts[:4]) if codec in ('vp9', 'av1', 'hvc1') else full_codec
+ if codec in ('dvh1', 'dvhe'):
+ hdr = 'DV'
+ elif codec == 'av1' and len(parts) > 3 and parts[3] == '10':
+ hdr = 'HDR10'
+ elif full_codec.replace('0', '').startswith('vp9.2'):
+ hdr = 'HDR10'
+ elif codec in ('flac', 'mp4a', 'opus', 'vorbis', 'mp3', 'aac', 'ac-3', 'ec-3', 'eac3', 'dtsc', 'dtse', 'dtsh', 'dtsl'):
if not acodec:
acodec = full_codec
+ elif codec in ('stpp', 'wvtt',):
+ if not tcodec:
+ tcodec = full_codec
else:
write_string('WARNING: Unknown codec %s\n' % full_codec, sys.stderr)
- if not vcodec and not acodec:
- if len(split_codecs) == 2:
- return {
- 'vcodec': split_codecs[0],
- 'acodec': split_codecs[1],
- }
- else:
+ if vcodec or acodec or tcodec:
return {
'vcodec': vcodec or 'none',
'acodec': acodec or 'none',
+ 'dynamic_range': hdr,
+ **({'tcodec': tcodec} if tcodec is not None else {}),
+ }
+ elif len(split_codecs) == 2:
+ return {
+ 'vcodec': split_codecs[0],
+ 'acodec': split_codecs[1],
}
return {}
if protocol is not None:
return protocol
- url = info_dict['url']
+ url = sanitize_url(info_dict['url'])
if url.startswith('rtmp'):
return 'rtmp'
elif url.startswith('mms'):
return compat_urllib_parse_urlparse(url).scheme
-def render_table(header_row, data, delim=False, extraGap=0, hideEmpty=False):
- """ Render a list of rows, each as a list of values """
+def render_table(header_row, data, delim=False, extra_gap=0, hide_empty=False):
+ """ Render a list of rows, each as a list of values.
+ Text after a \t will be right aligned """
+ def width(string):
+ return len(remove_terminal_sequences(string).replace('\t', ''))
def get_max_lens(table):
- return [max(len(compat_str(v)) for v in col) for col in zip(*table)]
+ return [max(width(str(v)) for v in col) for col in zip(*table)]
def filter_using_list(row, filterArray):
- return [col for (take, col) in zip(filterArray, row) if take]
+ return [col for take, col in itertools.zip_longest(filterArray, row, fillvalue=True) if take]
- if hideEmpty:
- max_lens = get_max_lens(data)
- header_row = filter_using_list(header_row, max_lens)
- data = [filter_using_list(row, max_lens) for row in data]
+ max_lens = get_max_lens(data) if hide_empty else []
+ header_row = filter_using_list(header_row, max_lens)
+ data = [filter_using_list(row, max_lens) for row in data]
table = [header_row] + data
max_lens = get_max_lens(table)
+ extra_gap += 1
if delim:
- table = [header_row] + [['-' * ml for ml in max_lens]] + data
- format_str = ' '.join('%-' + compat_str(ml + extraGap) + 's' for ml in max_lens[:-1]) + ' %s'
- return '\n'.join(format_str % tuple(row) for row in table)
+ table = [header_row, [delim * (ml + extra_gap) for ml in max_lens]] + data
+ table[1][-1] = table[1][-1][:-extra_gap] # Remove extra_gap from end of delimiter
+ for row in table:
+ for pos, text in enumerate(map(str, row)):
+ if '\t' in text:
+ row[pos] = text.replace('\t', ' ' * (max_lens[pos] - width(text))) + ' ' * extra_gap
+ else:
+ row[pos] = text + ' ' * (max_lens[pos] - width(text) + extra_gap)
+ ret = '\n'.join(''.join(row).rstrip() for row in table)
+ return ret
def _match_one(filter_part, dct, incomplete):
(?P<key>[a-z_]+)
\s*(?P<negation>!\s*)?(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
(?:
- (?P<intval>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)|
(?P<quote>["\'])(?P<quotedstrval>.+?)(?P=quote)|
(?P<strval>.+?)
)
''' % '|'.join(map(re.escape, COMPARISON_OPERATORS.keys())))
m = operator_rex.search(filter_part)
if m:
- unnegated_op = COMPARISON_OPERATORS[m.group('op')]
- if m.group('negation'):
+ m = m.groupdict()
+ unnegated_op = COMPARISON_OPERATORS[m['op']]
+ if m['negation']:
op = lambda attr, value: not unnegated_op(attr, value)
else:
op = unnegated_op
- actual_value = dct.get(m.group('key'))
- if (m.group('quotedstrval') is not None
- or m.group('strval') is not None
+ comparison_value = m['quotedstrval'] or m['strval'] or m['intval']
+ if m['quote']:
+ comparison_value = comparison_value.replace(r'\%s' % m['quote'], m['quote'])
+ actual_value = dct.get(m['key'])
+ numeric_comparison = None
+ if isinstance(actual_value, compat_numeric_types):
# If the original field is a string and matching comparisonvalue is
# a number we should respect the origin of the original field
# and process comparison value as a string (see
- # https://github.com/ytdl-org/youtube-dl/issues/11082).
- or actual_value is not None and m.group('intval') is not None
- and isinstance(actual_value, compat_str)):
- comparison_value = m.group('quotedstrval') or m.group('strval') or m.group('intval')
- quote = m.group('quote')
- if quote is not None:
- comparison_value = comparison_value.replace(r'\%s' % quote, quote)
- else:
- if m.group('op') in STRING_OPERATORS:
- raise ValueError('Operator %s only supports string values!' % m.group('op'))
+ # https://github.com/ytdl-org/youtube-dl/issues/11082)
try:
- comparison_value = int(m.group('intval'))
+ numeric_comparison = int(comparison_value)
except ValueError:
- comparison_value = parse_filesize(m.group('intval'))
- if comparison_value is None:
- comparison_value = parse_filesize(m.group('intval') + 'B')
- if comparison_value is None:
- raise ValueError(
- 'Invalid integer value %r in filter part %r' % (
- m.group('intval'), filter_part))
+ numeric_comparison = parse_filesize(comparison_value)
+ if numeric_comparison is None:
+ numeric_comparison = parse_filesize(f'{comparison_value}B')
+ if numeric_comparison is None:
+ numeric_comparison = parse_duration(comparison_value)
+ if numeric_comparison is not None and m['op'] in STRING_OPERATORS:
+ raise ValueError('Operator %s only supports string values!' % m['op'])
if actual_value is None:
- return incomplete or m.group('none_inclusive')
- return op(actual_value, comparison_value)
+ return incomplete or m['none_inclusive']
+ return op(actual_value, comparison_value if numeric_comparison is None else numeric_comparison)
UNARY_OPERATORS = {
'': lambda v: (v is True) if isinstance(v, bool) else (v is not None),
def srt_subtitles_timecode(seconds):
- return '%02d:%02d:%02d,%03d' % (seconds / 3600, (seconds % 3600) / 60, seconds % 60, (seconds % 1) * 1000)
+ return '%02d:%02d:%02d,%03d' % timetuple_from_msec(seconds * 1000)
+
+
+def ass_subtitles_timecode(seconds):
+ time = timetuple_from_msec(seconds * 1000)
+ return '%01d:%02d:%02d.%02d' % (*time[:-1], time.milliseconds / 10)
def dfxp2srt(dfxp_data):
+ [encodeFilename(path, True)])
try:
- p = subprocess.Popen(
+ p = Popen(
cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
except EnvironmentError as e:
raise XAttrMetadataError(e.errno, e.strerror)
- stdout, stderr = process_communicate_or_kill(p)
+ stdout, stderr = p.communicate_or_kill()
stderr = stderr.decode('utf-8', 'replace')
if p.returncode != 0:
raise XAttrMetadataError(p.returncode, stderr)
Icon=text-html
'''.lstrip()
+LINK_TEMPLATES = {
+ 'url': DOT_URL_LINK_TEMPLATE,
+ 'desktop': DOT_DESKTOP_LINK_TEMPLATE,
+ 'webloc': DOT_WEBLOC_LINK_TEMPLATE,
+}
+
def iri_to_uri(iri):
"""
def format_field(obj, field=None, template='%s', ignore=(None, ''), default='', func=None):
- if field is None:
- val = obj if obj is not None else default
- else:
- val = obj.get(field, default)
- if func and val not in ignore:
- val = func(val)
- return template % val if val not in ignore else default
+ val = traverse_obj(obj, *variadic(field))
+ if val in ignore:
+ return default
+ return template % (func(val) if func else val)
def clean_podcast_url(url):
def load_plugins(name, suffix, namespace):
- plugin_info = [None]
- classes = []
+ classes = {}
try:
- plugin_info = imp.find_module(
- name, [os.path.join(get_executable_path(), 'ytdlp_plugins')])
- plugins = imp.load_module(name, *plugin_info)
+ plugins_spec = importlib.util.spec_from_file_location(
+ name, os.path.join(get_executable_path(), 'ytdlp_plugins', name, '__init__.py'))
+ plugins = importlib.util.module_from_spec(plugins_spec)
+ sys.modules[plugins_spec.name] = plugins
+ plugins_spec.loader.exec_module(plugins)
for name in dir(plugins):
if name in namespace:
continue
if not name.endswith(suffix):
continue
klass = getattr(plugins, name)
- classes.append(klass)
- namespace[name] = klass
- except ImportError:
+ classes[name] = namespace[name] = klass
+ except FileNotFoundError:
pass
- finally:
- if plugin_info[0] is not None:
- plugin_info[0].close()
return classes
''' Traverse nested list/dict/tuple
@param path_list A list of paths which are checked one by one.
Each path is a list of keys where each key is a string,
- a tuple of strings or "...". When a tuple is given,
+ a function, a tuple of strings/None or "...".
+ When a fuction is given, it takes the key as argument and
+ returns whether the key matches or not. When a tuple is given,
all the keys given in the tuple are traversed, and
"..." traverses all the keys in the object
+ "None" returns the object without traversal
@param default Default value to return
@param expected_type Only accept final value of this type (Can also be any callable)
@param get_all Return all the values obtained from a path or only the first one
def _traverse_obj(obj, path, _current_depth=0):
nonlocal depth
- if obj is None:
- return None
path = tuple(variadic(path))
for i, key in enumerate(path):
+ if None in (key, obj):
+ return obj
if isinstance(key, (list, tuple)):
obj = [_traverse_obj(obj, sub_key, _current_depth) for sub_key in key]
key = ...
_current_depth += 1
depth = max(depth, _current_depth)
return [_traverse_obj(inner_obj, path[i + 1:], _current_depth) for inner_obj in obj]
+ elif callable(key):
+ if isinstance(obj, (list, tuple, LazyList)):
+ obj = enumerate(obj)
+ elif isinstance(obj, dict):
+ obj = obj.items()
+ else:
+ if not traverse_string:
+ return None
+ obj = str(obj)
+ _current_depth += 1
+ depth = max(depth, _current_depth)
+ return [_traverse_obj(v, path[i + 1:], _current_depth) for k, v in obj if key(k)]
elif isinstance(obj, dict) and not (is_user_input and key == ':'):
obj = (obj.get(key) if casesense or (key in obj)
else next((v for k, v in obj.items() if _lower(k) == key), None))
def traverse_dict(dictn, keys, casesense=True):
- ''' For backward compatibility. Do not use '''
- return traverse_obj(dictn, keys, casesense=casesense,
- is_user_input=True, traverse_string=True)
+ write_string('DeprecationWarning: yt_dlp.utils.traverse_dict is deprecated '
+ 'and may be removed in a future version. Use yt_dlp.utils.traverse_obj instead')
+ return traverse_obj(dictn, keys, casesense=casesense, is_user_input=True, traverse_string=True)
-def variadic(x, allowed_types=(str, bytes)):
+def variadic(x, allowed_types=(str, bytes, dict)):
return x if isinstance(x, collections.abc.Iterable) and not isinstance(x, allowed_types) else (x,)
signature_b64 = base64.b64encode(h.digest())
token = header_b64 + b'.' + payload_b64 + b'.' + signature_b64
return token
+
+
+# can be extended in future to verify the signature and parse header and return the algorithm used if it's not HS256
+def jwt_decode_hs256(jwt):
+ header_b64, payload_b64, signature_b64 = jwt.split('.')
+ payload_data = json.loads(base64.urlsafe_b64decode(payload_b64))
+ return payload_data
+
+
+def supports_terminal_sequences(stream):
+ if compat_os_name == 'nt':
+ from .compat import WINDOWS_VT_MODE # Must be imported locally
+ if not WINDOWS_VT_MODE or get_windows_version() < (10, 0, 10586):
+ return False
+ elif not os.getenv('TERM'):
+ return False
+ try:
+ return stream.isatty()
+ except BaseException:
+ return False
+
+
+_terminal_sequences_re = re.compile('\033\\[[^m]+m')
+
+
+def remove_terminal_sequences(string):
+ return _terminal_sequences_re.sub('', string)
+
+
+def number_of_digits(number):
+ return len('%d' % number)
+
+
+def join_nonempty(*values, delim='-', from_dict=None):
+ if from_dict is not None:
+ values = map(from_dict.get, values)
+ return delim.join(map(str, filter(None, values)))
+
+
+class Config:
+ own_args = None
+ filename = None
+ __initialized = False
+
+ def __init__(self, parser, label=None):
+ self._parser, self.label = parser, label
+ self._loaded_paths, self.configs = set(), []
+
+ def init(self, args=None, filename=None):
+ assert not self.__initialized
+ directory = ''
+ if filename:
+ location = os.path.realpath(filename)
+ directory = os.path.dirname(location)
+ if location in self._loaded_paths:
+ return False
+ self._loaded_paths.add(location)
+
+ self.__initialized = True
+ self.own_args, self.filename = args, filename
+ for location in self._parser.parse_args(args)[0].config_locations or []:
+ location = os.path.join(directory, expand_path(location))
+ if os.path.isdir(location):
+ location = os.path.join(location, 'yt-dlp.conf')
+ if not os.path.exists(location):
+ self._parser.error(f'config location {location} does not exist')
+ self.append_config(self.read_file(location), location)
+ return True
+
+ def __str__(self):
+ label = join_nonempty(
+ self.label, 'config', f'"{self.filename}"' if self.filename else '',
+ delim=' ')
+ return join_nonempty(
+ self.own_args is not None and f'{label[0].upper()}{label[1:]}: {self.hide_login_info(self.own_args)}',
+ *(f'\n{c}'.replace('\n', '\n| ')[1:] for c in self.configs),
+ delim='\n')
+
+ @staticmethod
+ def read_file(filename, default=[]):
+ try:
+ optionf = open(filename)
+ except IOError:
+ return default # silently skip if file is not present
+ try:
+ # FIXME: https://github.com/ytdl-org/youtube-dl/commit/dfe5fa49aed02cf36ba9f743b11b0903554b5e56
+ contents = optionf.read()
+ if sys.version_info < (3,):
+ contents = contents.decode(preferredencoding())
+ res = compat_shlex_split(contents, comments=True)
+ finally:
+ optionf.close()
+ return res
+
+ @staticmethod
+ def hide_login_info(opts):
+ PRIVATE_OPTS = set(['-p', '--password', '-u', '--username', '--video-password', '--ap-password', '--ap-username'])
+ eqre = re.compile('^(?P<key>' + ('|'.join(re.escape(po) for po in PRIVATE_OPTS)) + ')=.+$')
+
+ def _scrub_eq(o):
+ m = eqre.match(o)
+ if m:
+ return m.group('key') + '=PRIVATE'
+ else:
+ return o
+
+ opts = list(map(_scrub_eq, opts))
+ for idx, opt in enumerate(opts):
+ if opt in PRIVATE_OPTS and idx + 1 < len(opts):
+ opts[idx + 1] = 'PRIVATE'
+ return opts
+
+ def append_config(self, *args, label=None):
+ config = type(self)(self._parser, label)
+ config._loaded_paths = self._loaded_paths
+ if config.init(*args):
+ self.configs.append(config)
+
+ @property
+ def all_args(self):
+ for config in reversed(self.configs):
+ yield from config.all_args
+ yield from self.own_args or []
+
+ def parse_args(self):
+ return self._parser.parse_args(list(self.all_args))
projects / yt-dlp.git / blobdiff