2 from __future__
import unicode_literals
13 from .common
import InfoExtractor
14 from ..compat
import (
18 compat_urllib_parse_urlparse
,
32 return hashlib
.md5(text
.encode('utf-8')).hexdigest()
35 class IqiyiSDK(object):
36 def __init__(self
, target
, ip
, timestamp
):
39 self
.timestamp
= timestamp
43 return compat_str(sum(map(lambda p
: int(p
, 16), list(data
))))
47 if isinstance(num
, int):
49 return compat_str(sum(map(int, num
)))
52 even
= self
.digit_sum(compat_str(self
.timestamp
)[::2])
53 odd
= self
.digit_sum(compat_str(self
.timestamp
)[1::2])
56 def preprocess(self
, chunksize
):
57 self
.target
= md5_text(self
.target
)
59 for i
in range(32 // chunksize
):
60 chunks
.append(self
.target
[chunksize
* i
:chunksize
* (i
+ 1)])
62 chunks
.append(self
.target
[32 - 32 % chunksize
:])
63 return chunks
, list(map(int, self
.ip
.split('.')))
65 def mod(self
, modulus
):
66 chunks
, ip
= self
.preprocess(32)
67 self
.target
= chunks
[0] + ''.join(map(lambda p
: compat_str(p
% modulus
), ip
))
69 def split(self
, chunksize
):
76 chunks
, ip
= self
.preprocess(chunksize
)
78 for i
in range(len(chunks
)):
79 ip_part
= compat_str(ip
[i
] % modulus_map
[chunksize
]) if i
< 4 else ''
81 ret
+= ip_part
+ chunks
[i
]
83 ret
+= chunks
[i
] + ip_part
86 def handle_input16(self
):
87 self
.target
= md5_text(self
.target
)
88 self
.target
= self
.split_sum(self
.target
[:16]) + self
.target
+ self
.split_sum(self
.target
[16:])
90 def handle_input8(self
):
91 self
.target
= md5_text(self
.target
)
94 part
= self
.target
[8 * i
:8 * (i
+ 1)]
95 ret
+= self
.split_sum(part
) + part
99 self
.target
= md5_text(self
.target
)
100 self
.target
= self
.split_sum(self
.target
) + self
.target
102 def date(self
, scheme
):
103 self
.target
= md5_text(self
.target
)
104 d
= time
.localtime(self
.timestamp
)
106 'y': compat_str(d
.tm_year
),
107 'm': '%02d' % d
.tm_mon
,
108 'd': '%02d' % d
.tm_mday
,
110 self
.target
+= ''.join(map(lambda c
: strings
[c
], list(scheme
)))
112 def split_time_even_odd(self
):
113 even
, odd
= self
.even_odd()
114 self
.target
= odd
+ md5_text(self
.target
) + even
116 def split_time_odd_even(self
):
117 even
, odd
= self
.even_odd()
118 self
.target
= even
+ md5_text(self
.target
) + odd
120 def split_ip_time_sum(self
):
121 chunks
, ip
= self
.preprocess(32)
122 self
.target
= compat_str(sum(ip
)) + chunks
[0] + self
.digit_sum(self
.timestamp
)
124 def split_time_ip_sum(self
):
125 chunks
, ip
= self
.preprocess(32)
126 self
.target
= self
.digit_sum(self
.timestamp
) + chunks
[0] + compat_str(sum(ip
))
129 class IqiyiSDKInterpreter(object):
130 def __init__(self
, sdk_code
):
131 self
.sdk_code
= sdk_code
133 def decode_eval_codes(self
):
134 self
.sdk_code
= self
.sdk_code
[5:-3]
137 r
"'([^']+)',62,(\d+),'([^']+)'\.split\('\|'\),[^,]+,{}",
139 obfucasted_code
, count
, symbols
= mobj
.groups()
141 symbols
= symbols
.split('|')
146 b62count
= base_n(count
, 62)
147 symbol_table
[b62count
] = symbols
[count
] or b62count
149 self
.sdk_code
= re
.sub(
150 r
'\b(\w+)\b', lambda mobj
: symbol_table
[mobj
.group(0)],
153 def run(self
, target
, ip
, timestamp
):
154 self
.decode_eval_codes()
156 functions
= re
.findall(r
'input=([a-zA-Z0-9]+)\(input', self
.sdk_code
)
158 sdk
= IqiyiSDK(target
, ip
, timestamp
)
161 'handleSum': sdk
.handleSum
,
162 'handleInput8': sdk
.handle_input8
,
163 'handleInput16': sdk
.handle_input16
,
164 'splitTimeEvenOdd': sdk
.split_time_even_odd
,
165 'splitTimeOddEven': sdk
.split_time_odd_even
,
166 'splitIpTimeSum': sdk
.split_ip_time_sum
,
167 'splitTimeIpSum': sdk
.split_time_ip_sum
,
169 for function
in functions
:
170 if re
.match(r
'mod\d+', function
):
171 sdk
.mod(int(function
[3:]))
172 elif re
.match(r
'date[ymd]{3}', function
):
173 sdk
.date(function
[4:])
174 elif re
.match(r
'split\d+', function
):
175 sdk
.split(int(function
[5:]))
176 elif function
in other_functions
:
177 other_functions
[function
]()
179 raise ExtractorError('Unknown funcion %s' % function
)
184 class IqiyiIE(InfoExtractor
):
188 _VALID_URL
= r
'http://(?:[^.]+\.)?iqiyi\.com/.+\.html'
190 _NETRC_MACHINE
= 'iqiyi'
193 'url': 'http://www.iqiyi.com/v_19rrojlavg.html',
194 'md5': '2cb594dc2781e6c941a110d8f358118b',
196 'id': '9c1fb1b99d192b21c559e5a1a2cb3c73',
197 'title': '美国德州空中惊现奇异云团 酷似UFO',
201 'url': 'http://www.iqiyi.com/v_19rrhnnclk.html',
203 'id': 'e3f585b550a280af23c98b6cb2be19fb',
204 'title': '名侦探柯南第752集',
208 'id': 'e3f585b550a280af23c98b6cb2be19fb_part1',
210 'title': '名侦探柯南第752集',
214 'id': 'e3f585b550a280af23c98b6cb2be19fb_part2',
216 'title': '名侦探柯南第752集',
220 'id': 'e3f585b550a280af23c98b6cb2be19fb_part3',
222 'title': '名侦探柯南第752集',
226 'id': 'e3f585b550a280af23c98b6cb2be19fb_part4',
228 'title': '名侦探柯南第752集',
232 'id': 'e3f585b550a280af23c98b6cb2be19fb_part5',
234 'title': '名侦探柯南第752集',
238 'id': 'e3f585b550a280af23c98b6cb2be19fb_part6',
240 'title': '名侦探柯南第752集',
244 'id': 'e3f585b550a280af23c98b6cb2be19fb_part7',
246 'title': '名侦探柯南第752集',
250 'id': 'e3f585b550a280af23c98b6cb2be19fb_part8',
252 'title': '名侦探柯南第752集',
256 'skip_download': True,
259 'url': 'http://www.iqiyi.com/w_19rt6o8t9p.html',
260 'only_matching': True,
262 'url': 'http://www.iqiyi.com/a_19rrhbc6kt.html',
263 'only_matching': True,
265 'url': 'http://yule.iqiyi.com/pcb.html',
266 'only_matching': True,
268 # VIP-only video. The first 2 parts (6 minutes) are available without login
269 # MD5 sums omitted as values are different on Travis CI and my machine
270 'url': 'http://www.iqiyi.com/v_19rrny4w8w.html',
272 'id': 'f3cf468b39dddb30d676f89a91200dc1',
277 'id': 'f3cf468b39dddb30d676f89a91200dc1_part1',
283 'id': 'f3cf468b39dddb30d676f89a91200dc1_part2',
288 'expected_warnings': ['Needs a VIP account for full video'],
290 'url': 'http://www.iqiyi.com/a_19rrhb8ce1.html',
295 'playlist_count': 101,
307 def _real_initialize(self
):
312 # public key extracted from http://static.iqiyi.com/js/qiyiV2/20160129180840/jobs/i18n/i18nIndex.js
313 N
= 0xab86b6371b5318aaa1d3c9e612a9f1264f372323c8c0f19875b5fc3b3fd3afcc1e5bec527aa94bfa85bffc157e4245aebda05389a5357b75115ac94f074aefcd
316 return ohdave_rsa_encrypt(data
, e
, N
)
319 (username
, password
) = self
._get
_login
_info
()
321 # No authentication to be performed
325 data
= self
._download
_json
(
326 'http://kylin.iqiyi.com/get_token', None,
327 note
='Get token for logging', errnote
='Unable to get token for logging')
329 timestamp
= int(time
.time())
330 target
= '/apis/reglogin/login.action?lang=zh_TW&area_code=null&email=%s&passwd=%s&agenttype=1&from=undefined&keeplogin=0&piccode=&fromurl=&_pos=1' % (
331 username
, self
._rsa
_fun
(password
.encode('utf-8')))
333 interp
= IqiyiSDKInterpreter(sdk
)
334 sign
= interp
.run(target
, data
['ip'], timestamp
)
336 validation_params
= {
338 'server': 'BEA3AA1908656AABCCFF76582C4C6660',
339 'token': data
['token'],
340 'bird_src': 'f8d91d57af224da7893dd397d52d811a',
344 validation_result
= self
._download
_json
(
345 'http://kylin.iqiyi.com/validate?' + compat_urllib_parse
.urlencode(validation_params
), None,
346 note
='Validate credentials', errnote
='Unable to validate credentials')
349 'P00107': 'please login via the web interface and enter the CAPTCHA code',
350 'P00117': 'bad username or password',
353 code
= validation_result
['code']
355 msg
= MSG_MAP
.get(code
)
357 msg
= 'error %s' % code
358 if validation_result
.get('msg'):
359 msg
+= ': ' + validation_result
['msg']
360 self
._downloader
.report_warning('unable to log in: ' + msg
)
365 def _authenticate_vip_video(self
, api_video_url
, video_id
, tvid
, _uuid
, do_report_warning
):
367 # version and platform hard-coded in com/qiyi/player/core/model/remote/AuthenticationRemote.as
369 'platform': 'b6c13e26323c537d',
374 'playType': 'main', # XXX: always main?
375 'filename': os
.path
.splitext(url_basename(api_video_url
))[0],
378 qd_items
= compat_parse_qs(compat_urllib_parse_urlparse(api_video_url
).query
)
379 for key
, val
in qd_items
.items():
380 auth_params
[key
] = val
[0]
382 auth_req
= sanitized_Request(
383 'http://api.vip.iqiyi.com/services/ckn.action',
384 urlencode_postdata(auth_params
))
385 # iQiyi server throws HTTP 405 error without the following header
386 auth_req
.add_header('Content-Type', 'application/x-www-form-urlencoded')
387 auth_result
= self
._download
_json
(
389 note
='Downloading video authentication JSON',
390 errnote
='Unable to download video authentication JSON')
391 if auth_result
['code'] == 'Q00506': # requires a VIP account
392 if do_report_warning
:
393 self
.report_warning('Needs a VIP account for full video')
398 def construct_video_urls(self
, data
, video_id
, _uuid
, tvid
):
407 def get_encode_code(l
):
412 for i
in range(c
- 1, -1, -1):
413 a
= do_xor(int(b
[c
- i
- 1], 16), i
)
417 def get_path_key(x
, format_id
, segment_index
):
418 mg
= ')(*&^flash@#$%a'
419 tm
= self
._download
_json
(
420 'http://data.video.qiyi.com/t?tn=' + str(random
.random()), video_id
,
421 note
='Download path key of segment %d for format %s' % (segment_index
+ 1, format_id
)
423 t
= str(int(math
.floor(int(tm
) / (600.0))))
424 return md5_text(t
+ mg
+ x
)
427 need_vip_warning_report
= True
428 for format_item
in data
['vp']['tkl'][0]['vs']:
429 if 0 < int(format_item
['bid']) <= 10:
430 format_id
= self
.get_format(format_item
['bid'])
436 video_urls_info
= format_item
['fs']
437 if not format_item
['fs'][0]['l'].startswith('/'):
438 t
= get_encode_code(format_item
['fs'][0]['l'])
439 if t
.endswith('mp4'):
440 video_urls_info
= format_item
['flvs']
442 for segment_index
, segment
in enumerate(video_urls_info
):
444 if not vl
.startswith('/'):
445 vl
= get_encode_code(vl
)
446 is_vip_video
= '/vip/' in vl
447 filesize
= segment
['b']
448 base_url
= data
['vp']['du'].split('/')
451 vl
.split('/')[-1].split('.')[0], format_id
, segment_index
)
452 base_url
.insert(-1, key
)
453 base_url
= '/'.join(base_url
)
456 'qyid': uuid
.uuid4().hex,
461 'tn': str(int(time
.time()))
463 api_video_url
= base_url
+ vl
465 api_video_url
= api_video_url
.replace('.f4v', '.hml')
466 auth_result
= self
._authenticate
_vip
_video
(
467 api_video_url
, video_id
, tvid
, _uuid
, need_vip_warning_report
)
468 if auth_result
is False:
469 need_vip_warning_report
= False
472 't': auth_result
['data']['t'],
473 # cid is hard-coded in com/qiyi/player/core/player/RuntimeData.as
474 'cid': 'afbe8fd3d73448c9',
476 'QY00001': auth_result
['data']['u'],
478 api_video_url
+= '?' if '?' not in api_video_url
else '&'
479 api_video_url
+= compat_urllib_parse
.urlencode(param
)
480 js
= self
._download
_json
(
481 api_video_url
, video_id
,
482 note
='Download video info of segment %d for format %s' % (segment_index
+ 1, format_id
))
485 (video_url
, filesize
))
487 video_urls_dict
[format_id
] = video_urls
488 return video_urls_dict
490 def get_format(self
, bid
):
491 matched_format_ids
= [_format_id
for _bid
, _format_id
in self
._FORMATS
_MAP
if _bid
== str(bid
)]
492 return matched_format_ids
[0] if len(matched_format_ids
) else None
494 def get_bid(self
, format_id
):
495 matched_bids
= [_bid
for _bid
, _format_id
in self
._FORMATS
_MAP
if _format_id
== format_id
]
496 return matched_bids
[0] if len(matched_bids
) else None
498 def get_raw_data(self
, tvid
, video_id
, enc_key
, _uuid
):
499 tm
= str(int(time
.time()))
503 'src': md5_text('youtube-dl'),
508 'enc': md5_text(enc_key
+ tail
),
510 'tn': random
.random(),
512 'authkey': md5_text(md5_text('') + tail
),
516 api_url
= 'http://cache.video.qiyi.com/vms' + '?' + \
517 compat_urllib_parse
.urlencode(param
)
518 raw_data
= self
._download
_json
(api_url
, video_id
)
521 def get_enc_key(self
, swf_url
, video_id
):
522 # TODO: automatic key extraction
523 # last update at 2016-01-22 for Zombie::bite
524 enc_key
= '6ab6d0280511493ba85594779759d4ed'
527 def _extract_playlist(self
, webpage
):
531 r
'<a[^>]+class="site-piclist_pic_link"[^>]+href="(http://www\.iqiyi\.com/.+\.html)"',
536 album_id
= self
._search
_regex
(
537 r
'albumId\s*:\s*(\d+),', webpage
, 'album ID')
538 album_title
= self
._search
_regex
(
539 r
'data-share-title="([^"]+)"', webpage
, 'album title', fatal
=False)
541 entries
= list(map(self
.url_result
, links
))
543 # Start from 2 because links in the first page are already on webpage
544 for page_num
in itertools
.count(2):
545 pagelist_page
= self
._download
_webpage
(
546 'http://cache.video.qiyi.com/jp/avlist/%s/%d/%d/' % (album_id
, page_num
, PAGE_SIZE
),
548 note
='Download playlist page %d' % page_num
,
549 errnote
='Failed to download playlist page %d' % page_num
)
550 pagelist
= self
._parse
_json
(
551 remove_start(pagelist_page
, 'var tvInfoJs='), album_id
)
552 vlist
= pagelist
['data']['vlist']
554 entries
.append(self
.url_result(item
['vurl']))
555 if len(vlist
) < PAGE_SIZE
:
558 return self
.playlist_result(entries
, album_id
, album_title
)
560 def _real_extract(self
, url
):
561 webpage
= self
._download
_webpage
(
562 url
, 'temp_id', note
='download video page')
564 # There's no simple way to determine whether an URL is a playlist or not
566 playlist_result
= self
._extract
_playlist
(webpage
)
568 return playlist_result
570 tvid
= self
._search
_regex
(
571 r
'data-player-tvid\s*=\s*[\'"](\d+)', webpage, 'tvid')
572 video_id = self._search_regex(
573 r'data-player-videoid\s*=\s*[\'"]([a
-f\d
]+)', webpage, 'video_id
')
574 swf_url = self._search_regex(
575 r'(http
://[^
\'"]+MainPlayer[^.]+\.swf)', webpage, 'swf player URL')
576 _uuid = uuid.uuid4().hex
578 enc_key = self.get_enc_key(swf_url, video_id)
580 raw_data = self.get_raw_data(tvid, video_id, enc_key, _uuid)
582 if raw_data['code'] != 'A000000':
583 raise ExtractorError('Unable to load data. Error code: ' + raw_data['code'])
585 data = raw_data['data']
587 title = data['vi']['vn']
589 # generate video_urls_dict
590 video_urls_dict = self.construct_video_urls(
591 data, video_id, _uuid, tvid)
595 for format_id in video_urls_dict:
596 video_urls = video_urls_dict[format_id]
597 for i, video_url_info in enumerate(video_urls):
598 if len(entries) < i + 1:
599 entries.append({'formats': []})
600 entries[i]['formats'].append(
602 'url': video_url_info[0],
603 'filesize': video_url_info[-1],
604 'format_id': format_id,
605 'preference': int(self.get_bid(format_id))
609 for i in range(len(entries)):
610 self._sort_formats(entries[i]['formats'])
613 'id': '%s_part%d' % (video_id, i + 1),
620 '_type': 'multi_video',
627 info['id'] = video_id
628 info['title'] = title