yt_dlp/extractor/tencent.py

   1 import functools
   2 import random
   3 import re
   4 import string
   5 import time
   6
   7 from .common import InfoExtractor
   8 from ..aes import aes_cbc_encrypt_bytes
   9 from ..utils import (
  10     ExtractorError,
  11     determine_ext,
  12     float_or_none,
  13     int_or_none,
  14     js_to_json,
  15     traverse_obj,
  16     urljoin,
  17 )
  18
  19
  20 class TencentBaseIE(InfoExtractor):
  21     """Subclasses must set _API_URL, _APP_VERSION, _PLATFORM, _HOST, _REFERER"""
  22
  23     def _check_api_response(self, api_response):
  24         msg = api_response.get('msg')
  25         if api_response.get('code') != '0.0' and msg is not None:
  26             if msg in (
  27                 '您所在区域暂无此内容版权（如设置VPN请关闭后重试）',
  28                 'This content is not available in your area due to copyright restrictions. Please choose other videos.'
  29             ):
  30                 self.raise_geo_restricted()
  31             raise ExtractorError(f'Tencent said: {msg}')
  32
  33     def _get_ckey(self, video_id, url, guid):
  34         ua = self.get_param('http_headers')['User-Agent']
  35
  36         payload = (f'{video_id}|{int(time.time())}|mg3c3b04ba|{self._APP_VERSION}|{guid}|'
  37                    f'{self._PLATFORM}|{url[:48]}|{ua.lower()[:48]}||Mozilla|Netscape|Windows x86_64|00|')
  38
  39         return aes_cbc_encrypt_bytes(
  40             bytes(f'|{sum(map(ord, payload))}|{payload}', 'utf-8'),
  41             b'Ok\xda\xa3\x9e/\x8c\xb0\x7f^r-\x9e\xde\xf3\x14',
  42             b'\x01PJ\xf3V\xe6\x19\xcf.B\xbb\xa6\x8c?p\xf9',
  43             padding_mode='whitespace').hex().upper()
  44
  45     def _get_video_api_response(self, video_url, video_id, series_id, subtitle_format, video_format, video_quality):
  46         guid = ''.join(random.choices(string.digits + string.ascii_lowercase, k=16))
  47         ckey = self._get_ckey(video_id, video_url, guid)
  48         query = {
  49             'vid': video_id,
  50             'cid': series_id,
  51             'cKey': ckey,
  52             'encryptVer': '8.1',
  53             'spcaptiontype': '1' if subtitle_format == 'vtt' else '0',
  54             'sphls': '2' if video_format == 'hls' else '0',
  55             'dtype': '3' if video_format == 'hls' else '0',
  56             'defn': video_quality,
  57             'spsrt': '2',  # Enable subtitles
  58             'sphttps': '1',  # Enable HTTPS
  59             'otype': 'json',
  60             'spwm': '1',
  61             'hevclv': '28',  # Enable HEVC
  62             'drm': '40',  # Enable DRM
  63             # For HDR
  64             'spvideo': '4',
  65             'spsfrhdr': '100',
  66             # For SHD
  67             'host': self._HOST,
  68             'referer': self._REFERER,
  69             'ehost': video_url,
  70             'appVer': self._APP_VERSION,
  71             'platform': self._PLATFORM,
  72             # For VQQ
  73             'guid': guid,
  74             'flowid': ''.join(random.choices(string.digits + string.ascii_lowercase, k=32)),
  75         }
  76
  77         return self._search_json(r'QZOutputJson=', self._download_webpage(
  78             self._API_URL, video_id, query=query), 'api_response', video_id)
  79
  80     def _extract_video_formats_and_subtitles(self, api_response, video_id):
  81         video_response = api_response['vl']['vi'][0]
  82
  83         formats, subtitles = [], {}
  84         for video_format in video_response['ul']['ui']:
  85             if video_format.get('hls') or determine_ext(video_format['url']) == 'm3u8':
  86                 fmts, subs = self._extract_m3u8_formats_and_subtitles(
  87                     video_format['url'] + traverse_obj(video_format, ('hls', 'pt'), default=''),
  88                     video_id, 'mp4', fatal=False)
  89
  90                 formats.extend(fmts)
  91                 self._merge_subtitles(subs, target=subtitles)
  92             else:
  93                 formats.append({
  94                     'url': f'{video_format["url"]}{video_response["fn"]}?vkey={video_response["fvkey"]}',
  95                     'ext': 'mp4',
  96                 })
  97
  98         identifier = video_response.get('br')
  99         format_response = traverse_obj(
 100             api_response, ('fl', 'fi', lambda _, v: v['br'] == identifier),
 101             expected_type=dict, get_all=False) or {}
 102         common_info = {
 103             'width': video_response.get('vw'),
 104             'height': video_response.get('vh'),
 105             'abr': float_or_none(format_response.get('audiobandwidth'), scale=1000),
 106             'vbr': float_or_none(format_response.get('bandwidth'), scale=1000),
 107             'fps': format_response.get('vfps'),
 108             'format': format_response.get('sname'),
 109             'format_id': format_response.get('name'),
 110             'format_note': format_response.get('resolution'),
 111             'dynamic_range': {'hdr10': 'hdr10'}.get(format_response.get('name'), 'sdr'),
 112             'has_drm': format_response.get('drm', 0) != 0,
 113         }
 114         for f in formats:
 115             f.update(common_info)
 116
 117         return formats, subtitles
 118
 119     def _extract_video_native_subtitles(self, api_response):
 120         subtitles = {}
 121         for subtitle in traverse_obj(api_response, ('sfl', 'fi')) or ():
 122             subtitles.setdefault(subtitle['lang'].lower(), []).append({
 123                 'url': subtitle['url'],
 124                 'ext': 'srt' if subtitle.get('captionType') == 1 else 'vtt',
 125                 'protocol': 'm3u8_native' if determine_ext(subtitle['url']) == 'm3u8' else 'http',
 126             })
 127
 128         return subtitles
 129
 130     def _extract_all_video_formats_and_subtitles(self, url, video_id, series_id):
 131         api_responses = [self._get_video_api_response(url, video_id, series_id, 'srt', 'hls', 'hd')]
 132         self._check_api_response(api_responses[0])
 133         qualities = traverse_obj(api_responses, (0, 'fl', 'fi', ..., 'name')) or ('shd', 'fhd')
 134         for q in qualities:
 135             if q not in ('ld', 'sd', 'hd'):
 136                 api_responses.append(self._get_video_api_response(
 137                     url, video_id, series_id, 'vtt', 'hls', q))
 138                 self._check_api_response(api_responses[-1])
 139
 140         formats, subtitles = [], {}
 141         for api_response in api_responses:
 142             fmts, subs = self._extract_video_formats_and_subtitles(api_response, video_id)
 143             native_subtitles = self._extract_video_native_subtitles(api_response)
 144
 145             formats.extend(fmts)
 146             self._merge_subtitles(subs, native_subtitles, target=subtitles)
 147
 148         return formats, subtitles
 149
 150     def _get_clean_title(self, title):
 151         return re.sub(
 152             r'\s*[_\-]\s*(?:Watch online|Watch HD Video Online|WeTV|腾讯视频|(?:高清)?1080P在线观看平台).*?$',
 153             '', title or '').strip() or None
 154
 155
 156 class VQQBaseIE(TencentBaseIE):
 157     _VALID_URL_BASE = r'https?://v\.qq\.com'
 158
 159     _API_URL = 'https://h5vv6.video.qq.com/getvinfo'
 160     _APP_VERSION = '3.5.57'
 161     _PLATFORM = '10901'
 162     _HOST = 'v.qq.com'
 163     _REFERER = 'v.qq.com'
 164
 165     def _get_webpage_metadata(self, webpage, video_id):
 166         return self._search_json(
 167             r'<script[^>]*>[^<]*window\.__(?:pinia|PINIA__)\s*=',
 168             webpage, 'pinia data', video_id, transform_source=js_to_json, fatal=False)
 169
 170
 171 class VQQVideoIE(VQQBaseIE):
 172     IE_NAME = 'vqq:video'
 173     _VALID_URL = VQQBaseIE._VALID_URL_BASE + r'/x/(?:page|cover/(?P<series_id>\w+))/(?P<id>\w+)'
 174
 175     _TESTS = [{
 176         'url': 'https://v.qq.com/x/page/q326831cny0.html',
 177         'md5': 'b11c9cb781df710d686b950376676e2a',
 178         'info_dict': {
 179             'id': 'q326831cny0',
 180             'ext': 'mp4',
 181             'title': '我是选手：雷霆裂阵，终极时刻',
 182             'description': 'md5:e7ed70be89244017dac2a835a10aeb1e',
 183             'thumbnail': r're:^https?://[^?#]+q326831cny0',
 184             'format_id': r're:^shd',
 185         },
 186     }, {
 187         'url': 'https://v.qq.com/x/page/o3013za7cse.html',
 188         'md5': 'a1bcf42c6d28c189bd2fe2d468abb287',
 189         'info_dict': {
 190             'id': 'o3013za7cse',
 191             'ext': 'mp4',
 192             'title': '欧阳娜娜VLOG',
 193             'description': 'md5:29fe847497a98e04a8c3826e499edd2e',
 194             'thumbnail': r're:^https?://[^?#]+o3013za7cse',
 195             'format_id': r're:^shd',
 196         },
 197     }, {
 198         'url': 'https://v.qq.com/x/cover/7ce5noezvafma27/a00269ix3l8.html',
 199         'md5': '87968df6238a65d2478f19c25adf850b',
 200         'info_dict': {
 201             'id': 'a00269ix3l8',
 202             'ext': 'mp4',
 203             'title': '鸡毛飞上天 第01集',
 204             'description': 'md5:8cae3534327315b3872fbef5e51b5c5b',
 205             'thumbnail': r're:^https?://[^?#]+7ce5noezvafma27',
 206             'series': '鸡毛飞上天',
 207             'format_id': r're:^shd',
 208         },
 209         'skip': '404',
 210     }, {
 211         'url': 'https://v.qq.com/x/cover/mzc00200p29k31e/s0043cwsgj0.html',
 212         'md5': 'fadd10bf88aec3420f06f19ee1d24c5b',
 213         'info_dict': {
 214             'id': 's0043cwsgj0',
 215             'ext': 'mp4',
 216             'title': '第1集：如何快乐吃糖？',
 217             'description': 'md5:1d8c3a0b8729ae3827fa5b2d3ebd5213',
 218             'thumbnail': r're:^https?://[^?#]+s0043cwsgj0',
 219             'series': '青年理工工作者生活研究所',
 220             'format_id': r're:^shd',
 221         },
 222         'params': {'skip_download': 'm3u8'},
 223     }, {
 224         # Geo-restricted to China
 225         'url': 'https://v.qq.com/x/cover/mcv8hkc8zk8lnov/x0036x5qqsr.html',
 226         'only_matching': True,
 227     }]
 228
 229     def _real_extract(self, url):
 230         video_id, series_id = self._match_valid_url(url).group('id', 'series_id')
 231         webpage = self._download_webpage(url, video_id)
 232         webpage_metadata = self._get_webpage_metadata(webpage, video_id)
 233
 234         formats, subtitles = self._extract_all_video_formats_and_subtitles(url, video_id, series_id)
 235         return {
 236             'id': video_id,
 237             'title': self._get_clean_title(self._og_search_title(webpage)
 238                                            or traverse_obj(webpage_metadata, ('global', 'videoInfo', 'title'))),
 239             'description': (self._og_search_description(webpage)
 240                             or traverse_obj(webpage_metadata, ('global', 'videoInfo', 'desc'))),
 241             'formats': formats,
 242             'subtitles': subtitles,
 243             'thumbnail': (self._og_search_thumbnail(webpage)
 244                           or traverse_obj(webpage_metadata, ('global', 'videoInfo', 'pic160x90'))),
 245             'series': traverse_obj(webpage_metadata, ('global', 'coverInfo', 'title')),
 246         }
 247
 248
 249 class VQQSeriesIE(VQQBaseIE):
 250     IE_NAME = 'vqq:series'
 251     _VALID_URL = VQQBaseIE._VALID_URL_BASE + r'/x/cover/(?P<id>\w+)\.html/?(?:[?#]|$)'
 252
 253     _TESTS = [{
 254         'url': 'https://v.qq.com/x/cover/7ce5noezvafma27.html',
 255         'info_dict': {
 256             'id': '7ce5noezvafma27',
 257             'title': '鸡毛飞上天',
 258             'description': 'md5:8cae3534327315b3872fbef5e51b5c5b',
 259         },
 260         'playlist_count': 55,
 261     }, {
 262         'url': 'https://v.qq.com/x/cover/oshd7r0vy9sfq8e.html',
 263         'info_dict': {
 264             'id': 'oshd7r0vy9sfq8e',
 265             'title': '恋爱细胞2',
 266             'description': 'md5:9d8a2245679f71ca828534b0f95d2a03',
 267         },
 268         'playlist_count': 12,
 269     }]
 270
 271     def _real_extract(self, url):
 272         series_id = self._match_id(url)
 273         webpage = self._download_webpage(url, series_id)
 274         webpage_metadata = self._get_webpage_metadata(webpage, series_id)
 275
 276         episode_paths = [f'/x/cover/{series_id}/{video_id}.html' for video_id in re.findall(
 277             r'<div[^>]+data-vid="(?P<video_id>[^"]+)"[^>]+class="[^"]+episode-item-rect--number',
 278             webpage)]
 279
 280         return self.playlist_from_matches(
 281             episode_paths, series_id, ie=VQQVideoIE, getter=functools.partial(urljoin, url),
 282             title=self._get_clean_title(traverse_obj(webpage_metadata, ('coverInfo', 'title'))
 283                                         or self._og_search_title(webpage)),
 284             description=(traverse_obj(webpage_metadata, ('coverInfo', 'description'))
 285                          or self._og_search_description(webpage)))
 286
 287
 288 class WeTvBaseIE(TencentBaseIE):
 289     _VALID_URL_BASE = r'https?://(?:www\.)?wetv\.vip/(?:[^?#]+/)?play'
 290
 291     _API_URL = 'https://play.wetv.vip/getvinfo'
 292     _APP_VERSION = '3.5.57'
 293     _PLATFORM = '4830201'
 294     _HOST = 'wetv.vip'
 295     _REFERER = 'wetv.vip'
 296
 297     def _get_webpage_metadata(self, webpage, video_id):
 298         return self._parse_json(
 299             traverse_obj(self._search_nextjs_data(webpage, video_id), ('props', 'pageProps', 'data')),
 300             video_id, fatal=False)
 301
 302     def _extract_episode(self, url):
 303         video_id, series_id = self._match_valid_url(url).group('id', 'series_id')
 304         webpage = self._download_webpage(url, video_id)
 305         webpage_metadata = self._get_webpage_metadata(webpage, video_id)
 306
 307         formats, subtitles = self._extract_all_video_formats_and_subtitles(url, video_id, series_id)
 308         return {
 309             'id': video_id,
 310             'title': self._get_clean_title(self._og_search_title(webpage)
 311                                            or traverse_obj(webpage_metadata, ('coverInfo', 'title'))),
 312             'description': (traverse_obj(webpage_metadata, ('coverInfo', 'description'))
 313                             or self._og_search_description(webpage)),
 314             'formats': formats,
 315             'subtitles': subtitles,
 316             'thumbnail': self._og_search_thumbnail(webpage),
 317             'duration': int_or_none(traverse_obj(webpage_metadata, ('videoInfo', 'duration'))),
 318             'series': traverse_obj(webpage_metadata, ('coverInfo', 'title')),
 319             'episode_number': int_or_none(traverse_obj(webpage_metadata, ('videoInfo', 'episode'))),
 320         }
 321
 322     def _extract_series(self, url, ie):
 323         series_id = self._match_id(url)
 324         webpage = self._download_webpage(url, series_id)
 325         webpage_metadata = self._get_webpage_metadata(webpage, series_id)
 326
 327         episode_paths = ([f'/play/{series_id}/{episode["vid"]}' for episode in webpage_metadata.get('videoList')]
 328                          or re.findall(r'<a[^>]+class="play-video__link"[^>]+href="(?P<path>[^"]+)', webpage))
 329
 330         return self.playlist_from_matches(
 331             episode_paths, series_id, ie=ie, getter=functools.partial(urljoin, url),
 332             title=self._get_clean_title(traverse_obj(webpage_metadata, ('coverInfo', 'title'))
 333                                         or self._og_search_title(webpage)),
 334             description=(traverse_obj(webpage_metadata, ('coverInfo', 'description'))
 335                          or self._og_search_description(webpage)))
 336
 337
 338 class WeTvEpisodeIE(WeTvBaseIE):
 339     IE_NAME = 'wetv:episode'
 340     _VALID_URL = WeTvBaseIE._VALID_URL_BASE + r'/(?P<series_id>\w+)(?:-[^?#]+)?/(?P<id>\w+)(?:-[^?#]+)?'
 341
 342     _TESTS = [{
 343         'url': 'https://wetv.vip/en/play/air11ooo2rdsdi3-Cute-Programmer/v0040pr89t9-EP1-Cute-Programmer',
 344         'md5': '0c70fdfaa5011ab022eebc598e64bbbe',
 345         'info_dict': {
 346             'id': 'v0040pr89t9',
 347             'ext': 'mp4',
 348             'title': 'EP1: Cute Programmer',
 349             'description': 'md5:e87beab3bf9f392d6b9e541a63286343',
 350             'thumbnail': r're:^https?://[^?#]+air11ooo2rdsdi3',
 351             'series': 'Cute Programmer',
 352             'episode': 'Episode 1',
 353             'episode_number': 1,
 354             'duration': 2835,
 355             'format_id': r're:^shd',
 356         },
 357     }, {
 358         'url': 'https://wetv.vip/en/play/u37kgfnfzs73kiu/p0039b9nvik',
 359         'md5': '3b3c15ca4b9a158d8d28d5aa9d7c0a49',
 360         'info_dict': {
 361             'id': 'p0039b9nvik',
 362             'ext': 'mp4',
 363             'title': 'EP1: You Are My Glory',
 364             'description': 'md5:831363a4c3b4d7615e1f3854be3a123b',
 365             'thumbnail': r're:^https?://[^?#]+u37kgfnfzs73kiu',
 366             'series': 'You Are My Glory',
 367             'episode': 'Episode 1',
 368             'episode_number': 1,
 369             'duration': 2454,
 370             'format_id': r're:^shd',
 371         },
 372     }, {
 373         'url': 'https://wetv.vip/en/play/lcxgwod5hapghvw-WeTV-PICK-A-BOO/i0042y00lxp-Zhao-Lusi-Describes-The-First-Experiences-She-Had-In-Who-Rules-The-World-%7C-WeTV-PICK-A-BOO',
 374         'md5': '71133f5c2d5d6cad3427e1b010488280',
 375         'info_dict': {
 376             'id': 'i0042y00lxp',
 377             'ext': 'mp4',
 378             'title': 'md5:f7a0857dbe5fbbe2e7ad630b92b54e6a',
 379             'description': 'md5:76260cb9cdc0ef76826d7ca9d92fadfa',
 380             'thumbnail': r're:^https?://[^?#]+i0042y00lxp',
 381             'series': 'WeTV PICK-A-BOO',
 382             'episode': 'Episode 0',
 383             'episode_number': 0,
 384             'duration': 442,
 385             'format_id': r're:^shd',
 386         },
 387     }]
 388
 389     def _real_extract(self, url):
 390         return self._extract_episode(url)
 391
 392
 393 class WeTvSeriesIE(WeTvBaseIE):
 394     _VALID_URL = WeTvBaseIE._VALID_URL_BASE + r'/(?P<id>\w+)(?:-[^/?#]+)?/?(?:[?#]|$)'
 395
 396     _TESTS = [{
 397         'url': 'https://wetv.vip/play/air11ooo2rdsdi3-Cute-Programmer',
 398         'info_dict': {
 399             'id': 'air11ooo2rdsdi3',
 400             'title': 'Cute Programmer',
 401             'description': 'md5:e87beab3bf9f392d6b9e541a63286343',
 402         },
 403         'playlist_count': 30,
 404     }, {
 405         'url': 'https://wetv.vip/en/play/u37kgfnfzs73kiu-You-Are-My-Glory',
 406         'info_dict': {
 407             'id': 'u37kgfnfzs73kiu',
 408             'title': 'You Are My Glory',
 409             'description': 'md5:831363a4c3b4d7615e1f3854be3a123b',
 410         },
 411         'playlist_count': 32,
 412     }]
 413
 414     def _real_extract(self, url):
 415         return self._extract_series(url, WeTvEpisodeIE)
 416
 417
 418 class IflixBaseIE(WeTvBaseIE):
 419     _VALID_URL_BASE = r'https?://(?:www\.)?iflix\.com/(?:[^?#]+/)?play'
 420
 421     _API_URL = 'https://vplay.iflix.com/getvinfo'
 422     _APP_VERSION = '3.5.57'
 423     _PLATFORM = '330201'
 424     _HOST = 'www.iflix.com'
 425     _REFERER = 'www.iflix.com'
 426
 427
 428 class IflixEpisodeIE(IflixBaseIE):
 429     IE_NAME = 'iflix:episode'
 430     _VALID_URL = IflixBaseIE._VALID_URL_BASE + r'/(?P<series_id>\w+)(?:-[^?#]+)?/(?P<id>\w+)(?:-[^?#]+)?'
 431
 432     _TESTS = [{
 433         'url': 'https://www.iflix.com/en/play/daijrxu03yypu0s/a0040kvgaza',
 434         'md5': '9740f9338c3a2105290d16b68fb3262f',
 435         'info_dict': {
 436             'id': 'a0040kvgaza',
 437             'ext': 'mp4',
 438             'title': 'EP1: Put Your Head On My Shoulder 2021',
 439             'description': 'md5:c095a742d3b7da6dfedd0c8170727a42',
 440             'thumbnail': r're:^https?://[^?#]+daijrxu03yypu0s',
 441             'series': 'Put Your Head On My Shoulder 2021',
 442             'episode': 'Episode 1',
 443             'episode_number': 1,
 444             'duration': 2639,
 445             'format_id': r're:^shd',
 446         },
 447     }, {
 448         'url': 'https://www.iflix.com/en/play/fvvrcc3ra9lbtt1-Take-My-Brother-Away/i0029sd3gm1-EP1%EF%BC%9ATake-My-Brother-Away',
 449         'md5': '375c9b8478fdedca062274b2c2f53681',
 450         'info_dict': {
 451             'id': 'i0029sd3gm1',
 452             'ext': 'mp4',
 453             'title': 'EP1：Take My Brother Away',
 454             'description': 'md5:f0f7be1606af51cd94d5627de96b0c76',
 455             'thumbnail': r're:^https?://[^?#]+fvvrcc3ra9lbtt1',
 456             'series': 'Take My Brother Away',
 457             'episode': 'Episode 1',
 458             'episode_number': 1,
 459             'duration': 228,
 460             'format_id': r're:^shd',
 461         },
 462     }]
 463
 464     def _real_extract(self, url):
 465         return self._extract_episode(url)
 466
 467
 468 class IflixSeriesIE(IflixBaseIE):
 469     _VALID_URL = IflixBaseIE._VALID_URL_BASE + r'/(?P<id>\w+)(?:-[^/?#]+)?/?(?:[?#]|$)'
 470
 471     _TESTS = [{
 472         'url': 'https://www.iflix.com/en/play/g21a6qk4u1s9x22-You-Are-My-Hero',
 473         'info_dict': {
 474             'id': 'g21a6qk4u1s9x22',
 475             'title': 'You Are My Hero',
 476             'description': 'md5:9c4d844bc0799cd3d2b5aed758a2050a',
 477         },
 478         'playlist_count': 40,
 479     }, {
 480         'url': 'https://www.iflix.com/play/0s682hc45t0ohll',
 481         'info_dict': {
 482             'id': '0s682hc45t0ohll',
 483             'title': 'Miss Gu Who Is Silent',
 484             'description': 'md5:a9651d0236f25af06435e845fa2f8c78',
 485         },
 486         'playlist_count': 20,
 487     }]
 488
 489     def _real_extract(self, url):
 490         return self._extract_series(url, IflixEpisodeIE)