youtube_dl/extractor/iqiyi.py

   1 # coding: utf-8
   2 from __future__ import unicode_literals
   3
   4 import hashlib
   5 import math
   6 import os
   7 import random
   8 import time
   9 import uuid
  10
  11 from .common import InfoExtractor
  12 from ..compat import (
  13     compat_parse_qs,
  14     compat_urllib_parse,
  15     compat_urllib_parse_urlparse,
  16 )
  17 from ..utils import (
  18     ExtractorError,
  19     sanitized_Request,
  20     urlencode_postdata,
  21     url_basename,
  22 )
  23
  24
  25 class IqiyiIE(InfoExtractor):
  26     IE_NAME = 'iqiyi'
  27     IE_DESC = '爱奇艺'
  28
  29     _VALID_URL = r'http://(?:[^.]+\.)?iqiyi\.com/.+\.html'
  30
  31     _TESTS = [{
  32         'url': 'http://www.iqiyi.com/v_19rrojlavg.html',
  33         'md5': '2cb594dc2781e6c941a110d8f358118b',
  34         'info_dict': {
  35             'id': '9c1fb1b99d192b21c559e5a1a2cb3c73',
  36             'title': '美国德州空中惊现奇异云团 酷似UFO',
  37             'ext': 'f4v',
  38         }
  39     }, {
  40         'url': 'http://www.iqiyi.com/v_19rrhnnclk.html',
  41         'info_dict': {
  42             'id': 'e3f585b550a280af23c98b6cb2be19fb',
  43             'title': '名侦探柯南第752集',
  44         },
  45         'playlist': [{
  46             'info_dict': {
  47                 'id': 'e3f585b550a280af23c98b6cb2be19fb_part1',
  48                 'ext': 'f4v',
  49                 'title': '名侦探柯南第752集',
  50             },
  51         }, {
  52             'info_dict': {
  53                 'id': 'e3f585b550a280af23c98b6cb2be19fb_part2',
  54                 'ext': 'f4v',
  55                 'title': '名侦探柯南第752集',
  56             },
  57         }, {
  58             'info_dict': {
  59                 'id': 'e3f585b550a280af23c98b6cb2be19fb_part3',
  60                 'ext': 'f4v',
  61                 'title': '名侦探柯南第752集',
  62             },
  63         }, {
  64             'info_dict': {
  65                 'id': 'e3f585b550a280af23c98b6cb2be19fb_part4',
  66                 'ext': 'f4v',
  67                 'title': '名侦探柯南第752集',
  68             },
  69         }, {
  70             'info_dict': {
  71                 'id': 'e3f585b550a280af23c98b6cb2be19fb_part5',
  72                 'ext': 'f4v',
  73                 'title': '名侦探柯南第752集',
  74             },
  75         }, {
  76             'info_dict': {
  77                 'id': 'e3f585b550a280af23c98b6cb2be19fb_part6',
  78                 'ext': 'f4v',
  79                 'title': '名侦探柯南第752集',
  80             },
  81         }, {
  82             'info_dict': {
  83                 'id': 'e3f585b550a280af23c98b6cb2be19fb_part7',
  84                 'ext': 'f4v',
  85                 'title': '名侦探柯南第752集',
  86             },
  87         }, {
  88             'info_dict': {
  89                 'id': 'e3f585b550a280af23c98b6cb2be19fb_part8',
  90                 'ext': 'f4v',
  91                 'title': '名侦探柯南第752集',
  92             },
  93         }],
  94         'params': {
  95             'skip_download': True,
  96         },
  97     }, {
  98         'url': 'http://www.iqiyi.com/w_19rt6o8t9p.html',
  99         'only_matching': True,
 100     }, {
 101         'url': 'http://www.iqiyi.com/a_19rrhbc6kt.html',
 102         'only_matching': True,
 103     }, {
 104         'url': 'http://yule.iqiyi.com/pcb.html',
 105         'only_matching': True,
 106     }, {
 107         # VIP-only video. The first 2 parts (6 minutes) are available without login
 108         # MD5 sums omitted as values are different on Travis CI and my machine
 109         'url': 'http://www.iqiyi.com/v_19rrny4w8w.html',
 110         'info_dict': {
 111             'id': 'f3cf468b39dddb30d676f89a91200dc1',
 112             'title': '泰坦尼克号',
 113         },
 114         'playlist': [{
 115             'info_dict': {
 116                 'id': 'f3cf468b39dddb30d676f89a91200dc1_part1',
 117                 'ext': 'f4v',
 118                 'title': '泰坦尼克号',
 119             },
 120         }, {
 121             'info_dict': {
 122                 'id': 'f3cf468b39dddb30d676f89a91200dc1_part2',
 123                 'ext': 'f4v',
 124                 'title': '泰坦尼克号',
 125             },
 126         }],
 127         'expected_warnings': ['Needs a VIP account for full video'],
 128     }]
 129
 130     _FORMATS_MAP = [
 131         ('1', 'h6'),
 132         ('2', 'h5'),
 133         ('3', 'h4'),
 134         ('4', 'h3'),
 135         ('5', 'h2'),
 136         ('10', 'h1'),
 137     ]
 138
 139     @staticmethod
 140     def md5_text(text):
 141         return hashlib.md5(text.encode('utf-8')).hexdigest()
 142
 143     def _authenticate_vip_video(self, api_video_url, video_id, tvid, _uuid, do_report_warning):
 144         auth_params = {
 145             # version and platform hard-coded in com/qiyi/player/core/model/remote/AuthenticationRemote.as
 146             'version': '2.0',
 147             'platform': 'b6c13e26323c537d',
 148             'aid': tvid,
 149             'tvid': tvid,
 150             'uid': '',
 151             'deviceId': _uuid,
 152             'playType': 'main',  # XXX: always main?
 153             'filename': os.path.splitext(url_basename(api_video_url))[0],
 154         }
 155
 156         qd_items = compat_parse_qs(compat_urllib_parse_urlparse(api_video_url).query)
 157         for key, val in qd_items.items():
 158             auth_params[key] = val[0]
 159
 160         auth_req = sanitized_Request(
 161             'http://api.vip.iqiyi.com/services/ckn.action',
 162             urlencode_postdata(auth_params))
 163         # iQiyi server throws HTTP 405 error without the following header
 164         auth_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
 165         auth_result = self._download_json(
 166             auth_req, video_id,
 167             note='Downloading video authentication JSON',
 168             errnote='Unable to download video authentication JSON')
 169         if auth_result['code'] == 'Q00506':  # requires a VIP account
 170             if do_report_warning:
 171                 self.report_warning('Needs a VIP account for full video')
 172             return False
 173
 174         return auth_result
 175
 176     def construct_video_urls(self, data, video_id, _uuid, tvid):
 177         def do_xor(x, y):
 178             a = y % 3
 179             if a == 1:
 180                 return x ^ 121
 181             if a == 2:
 182                 return x ^ 72
 183             return x ^ 103
 184
 185         def get_encode_code(l):
 186             a = 0
 187             b = l.split('-')
 188             c = len(b)
 189             s = ''
 190             for i in range(c - 1, -1, -1):
 191                 a = do_xor(int(b[c - i - 1], 16), i)
 192                 s += chr(a)
 193             return s[::-1]
 194
 195         def get_path_key(x, format_id, segment_index):
 196             mg = ')(*&^flash@#$%a'
 197             tm = self._download_json(
 198                 'http://data.video.qiyi.com/t?tn=' + str(random.random()), video_id,
 199                 note='Download path key of segment %d for format %s' % (segment_index + 1, format_id)
 200             )['t']
 201             t = str(int(math.floor(int(tm) / (600.0))))
 202             return self.md5_text(t + mg + x)
 203
 204         video_urls_dict = {}
 205         need_vip_warning_report = True
 206         for format_item in data['vp']['tkl'][0]['vs']:
 207             if 0 < int(format_item['bid']) <= 10:
 208                 format_id = self.get_format(format_item['bid'])
 209             else:
 210                 continue
 211
 212             video_urls = []
 213
 214             video_urls_info = format_item['fs']
 215             if not format_item['fs'][0]['l'].startswith('/'):
 216                 t = get_encode_code(format_item['fs'][0]['l'])
 217                 if t.endswith('mp4'):
 218                     video_urls_info = format_item['flvs']
 219
 220             for segment_index, segment in enumerate(video_urls_info):
 221                 vl = segment['l']
 222                 if not vl.startswith('/'):
 223                     vl = get_encode_code(vl)
 224                 is_vip_video = '/vip/' in vl
 225                 filesize = segment['b']
 226                 base_url = data['vp']['du'].split('/')
 227                 if not is_vip_video:
 228                     key = get_path_key(
 229                         vl.split('/')[-1].split('.')[0], format_id, segment_index)
 230                     base_url.insert(-1, key)
 231                 base_url = '/'.join(base_url)
 232                 param = {
 233                     'su': _uuid,
 234                     'qyid': uuid.uuid4().hex,
 235                     'client': '',
 236                     'z': '',
 237                     'bt': '',
 238                     'ct': '',
 239                     'tn': str(int(time.time()))
 240                 }
 241                 api_video_url = base_url + vl
 242                 if is_vip_video:
 243                     api_video_url = api_video_url.replace('.f4v', '.hml')
 244                     auth_result = self._authenticate_vip_video(
 245                         api_video_url, video_id, tvid, _uuid, need_vip_warning_report)
 246                     if auth_result is False:
 247                         need_vip_warning_report = False
 248                         break
 249                     param.update({
 250                         't': auth_result['data']['t'],
 251                         # cid is hard-coded in com/qiyi/player/core/player/RuntimeData.as
 252                         'cid': 'afbe8fd3d73448c9',
 253                         'vid': video_id,
 254                         'QY00001': auth_result['data']['u'],
 255                     })
 256                 api_video_url += '?' if '?' not in api_video_url else '&'
 257                 api_video_url += compat_urllib_parse.urlencode(param)
 258                 js = self._download_json(
 259                     api_video_url, video_id,
 260                     note='Download video info of segment %d for format %s' % (segment_index + 1, format_id))
 261                 video_url = js['l']
 262                 video_urls.append(
 263                     (video_url, filesize))
 264
 265             video_urls_dict[format_id] = video_urls
 266         return video_urls_dict
 267
 268     def get_format(self, bid):
 269         matched_format_ids = [_format_id for _bid, _format_id in self._FORMATS_MAP if _bid == str(bid)]
 270         return matched_format_ids[0] if len(matched_format_ids) else None
 271
 272     def get_bid(self, format_id):
 273         matched_bids = [_bid for _bid, _format_id in self._FORMATS_MAP if _format_id == format_id]
 274         return matched_bids[0] if len(matched_bids) else None
 275
 276     def get_raw_data(self, tvid, video_id, enc_key, _uuid):
 277         tm = str(int(time.time()))
 278         tail = tm + tvid
 279         param = {
 280             'key': 'fvip',
 281             'src': self.md5_text('youtube-dl'),
 282             'tvId': tvid,
 283             'vid': video_id,
 284             'vinfo': 1,
 285             'tm': tm,
 286             'enc': self.md5_text(enc_key + tail),
 287             'qyid': _uuid,
 288             'tn': random.random(),
 289             'um': 0,
 290             'authkey': self.md5_text(self.md5_text('') + tail),
 291             'k_tag': 1,
 292         }
 293
 294         api_url = 'http://cache.video.qiyi.com/vms' + '?' + \
 295             compat_urllib_parse.urlencode(param)
 296         raw_data = self._download_json(api_url, video_id)
 297         return raw_data
 298
 299     def get_enc_key(self, swf_url, video_id):
 300         # TODO: automatic key extraction
 301         # last update at 2016-01-22 for Zombie::bite
 302         enc_key = '6ab6d0280511493ba85594779759d4ed'
 303         return enc_key
 304
 305     def _real_extract(self, url):
 306         webpage = self._download_webpage(
 307             url, 'temp_id', note='download video page')
 308         tvid = self._search_regex(
 309             r'data-player-tvid\s*=\s*[\'"](\d+)', webpage, 'tvid')
 310         video_id = self._search_regex(
 311             r'data-player-videoid\s*=\s*[\'"]([a-f\d]+)', webpage, 'video_id')
 312         swf_url = self._search_regex(
 313             r'(http://[^\'"]+MainPlayer[^.]+\.swf)', webpage, 'swf player URL')
 314         _uuid = uuid.uuid4().hex
 315
 316         enc_key = self.get_enc_key(swf_url, video_id)
 317
 318         raw_data = self.get_raw_data(tvid, video_id, enc_key, _uuid)
 319
 320         if raw_data['code'] != 'A000000':
 321             raise ExtractorError('Unable to load data. Error code: ' + raw_data['code'])
 322
 323         data = raw_data['data']
 324
 325         title = data['vi']['vn']
 326
 327         # generate video_urls_dict
 328         video_urls_dict = self.construct_video_urls(
 329             data, video_id, _uuid, tvid)
 330
 331         # construct info
 332         entries = []
 333         for format_id in video_urls_dict:
 334             video_urls = video_urls_dict[format_id]
 335             for i, video_url_info in enumerate(video_urls):
 336                 if len(entries) < i + 1:
 337                     entries.append({'formats': []})
 338                 entries[i]['formats'].append(
 339                     {
 340                         'url': video_url_info[0],
 341                         'filesize': video_url_info[-1],
 342                         'format_id': format_id,
 343                         'preference': int(self.get_bid(format_id))
 344                     }
 345                 )
 346
 347         for i in range(len(entries)):
 348             self._sort_formats(entries[i]['formats'])
 349             entries[i].update(
 350                 {
 351                     'id': '%s_part%d' % (video_id, i + 1),
 352                     'title': title,
 353                 }
 354             )
 355
 356         if len(entries) > 1:
 357             info = {
 358                 '_type': 'multi_video',
 359                 'id': video_id,
 360                 'title': title,
 361                 'entries': entries,
 362             }
 363         else:
 364             info = entries[0]
 365             info['id'] = video_id
 366             info['title'] = title
 367
 368         return info