youtube_dl/extractor/youku.py

   1 # coding: utf-8
   2 from __future__ import unicode_literals
   3
   4 import base64
   5 import random
   6 import string
   7 import time
   8
   9 from .common import InfoExtractor
  10 from ..compat import (
  11     compat_urllib_parse_urlencode,
  12     compat_ord,
  13 )
  14 from ..utils import (
  15     ExtractorError,
  16     sanitized_Request,
  17 )
  18
  19
  20 class YoukuIE(InfoExtractor):
  21     IE_NAME = 'youku'
  22     IE_DESC = '优酷'
  23     _VALID_URL = r'''(?x)
  24         (?:
  25             http://(?:v|player)\.youku\.com/(?:v_show/id_|player\.php/sid/)|
  26             youku:)
  27         (?P<id>[A-Za-z0-9]+)(?:\.html|/v\.swf|)
  28     '''
  29
  30     _TESTS = [{
  31         # MD5 is unstable
  32         'url': 'http://v.youku.com/v_show/id_XMTc1ODE5Njcy.html',
  33         'info_dict': {
  34             'id': 'XMTc1ODE5Njcy_part1',
  35             'title': '★Smile﹗♡ Git Fresh -Booty Music舞蹈.',
  36             'ext': 'flv'
  37         }
  38     }, {
  39         'url': 'http://player.youku.com/player.php/sid/XNDgyMDQ2NTQw/v.swf',
  40         'only_matching': True,
  41     }, {
  42         'url': 'http://v.youku.com/v_show/id_XODgxNjg1Mzk2_ev_1.html',
  43         'info_dict': {
  44             'id': 'XODgxNjg1Mzk2',
  45             'title': '武媚娘传奇 85',
  46         },
  47         'playlist_count': 11,
  48         'skip': 'Available in China only',
  49     }, {
  50         'url': 'http://v.youku.com/v_show/id_XMTI1OTczNDM5Mg==.html',
  51         'info_dict': {
  52             'id': 'XMTI1OTczNDM5Mg',
  53             'title': '花千骨 04',
  54         },
  55         'playlist_count': 13,
  56     }, {
  57         'url': 'http://v.youku.com/v_show/id_XNjA1NzA2Njgw.html',
  58         'note': 'Video protected with password',
  59         'info_dict': {
  60             'id': 'XNjA1NzA2Njgw',
  61             'title': '邢義田复旦讲座之想象中的胡人—从“左衽孔子”说起',
  62         },
  63         'playlist_count': 19,
  64         'params': {
  65             'videopassword': '100600',
  66         },
  67     }, {
  68         # /play/get.json contains streams with "channel_type":"tail"
  69         'url': 'http://v.youku.com/v_show/id_XOTUxMzg4NDMy.html',
  70         'info_dict': {
  71             'id': 'XOTUxMzg4NDMy',
  72             'title': '我的世界☆明月庄主☆车震猎杀☆杀人艺术Minecraft',
  73         },
  74         'playlist_count': 6,
  75     }]
  76
  77     def construct_video_urls(self, data):
  78         # get sid, token
  79         def yk_t(s1, s2):
  80             ls = list(range(256))
  81             t = 0
  82             for i in range(256):
  83                 t = (t + ls[i] + compat_ord(s1[i % len(s1)])) % 256
  84                 ls[i], ls[t] = ls[t], ls[i]
  85             s = bytearray()
  86             x, y = 0, 0
  87             for i in range(len(s2)):
  88                 y = (y + 1) % 256
  89                 x = (x + ls[y]) % 256
  90                 ls[x], ls[y] = ls[y], ls[x]
  91                 s.append(compat_ord(s2[i]) ^ ls[(ls[x] + ls[y]) % 256])
  92             return bytes(s)
  93
  94         sid, token = yk_t(
  95             b'becaf9be', base64.b64decode(data['security']['encrypt_string'].encode('ascii'))
  96         ).decode('ascii').split('_')
  97
  98         # get oip
  99         oip = data['security']['ip']
 100
 101         fileid_dict = {}
 102         for stream in data['stream']:
 103             if stream.get('channel_type') == 'tail':
 104                 continue
 105             format = stream.get('stream_type')
 106             fileid = stream['stream_fileid']
 107             fileid_dict[format] = fileid
 108
 109         def get_fileid(format, n):
 110             number = hex(int(str(n), 10))[2:].upper()
 111             if len(number) == 1:
 112                 number = '0' + number
 113             streamfileids = fileid_dict[format]
 114             fileid = streamfileids[0:8] + number + streamfileids[10:]
 115             return fileid
 116
 117         # get ep
 118         def generate_ep(format, n):
 119             fileid = get_fileid(format, n)
 120             ep_t = yk_t(
 121                 b'bf7e5f01',
 122                 ('%s_%s_%s' % (sid, fileid, token)).encode('ascii')
 123             )
 124             ep = base64.b64encode(ep_t).decode('ascii')
 125             return ep
 126
 127         # generate video_urls
 128         video_urls_dict = {}
 129         for stream in data['stream']:
 130             if stream.get('channel_type') == 'tail':
 131                 continue
 132             format = stream.get('stream_type')
 133             video_urls = []
 134             for dt in stream['segs']:
 135                 n = str(stream['segs'].index(dt))
 136                 param = {
 137                     'K': dt['key'],
 138                     'hd': self.get_hd(format),
 139                     'myp': 0,
 140                     'ypp': 0,
 141                     'ctype': 12,
 142                     'ev': 1,
 143                     'token': token,
 144                     'oip': oip,
 145                     'ep': generate_ep(format, n)
 146                 }
 147                 video_url = \
 148                     'http://k.youku.com/player/getFlvPath/' + \
 149                     'sid/' + sid + \
 150                     '_00' + \
 151                     '/st/' + self.parse_ext_l(format) + \
 152                     '/fileid/' + get_fileid(format, n) + '?' + \
 153                     compat_urllib_parse_urlencode(param)
 154                 video_urls.append(video_url)
 155             video_urls_dict[format] = video_urls
 156
 157         return video_urls_dict
 158
 159     @staticmethod
 160     def get_ysuid():
 161         return '%d%s' % (int(time.time()), ''.join([
 162             random.choice(string.ascii_letters) for i in range(3)]))
 163
 164     def get_hd(self, fm):
 165         hd_id_dict = {
 166             '3gp': '0',
 167             '3gphd': '1',
 168             'flv': '0',
 169             'flvhd': '0',
 170             'mp4': '1',
 171             'mp4hd': '1',
 172             'mp4hd2': '1',
 173             'mp4hd3': '1',
 174             'hd2': '2',
 175             'hd3': '3',
 176         }
 177         return hd_id_dict[fm]
 178
 179     def parse_ext_l(self, fm):
 180         ext_dict = {
 181             '3gp': 'flv',
 182             '3gphd': 'mp4',
 183             'flv': 'flv',
 184             'flvhd': 'flv',
 185             'mp4': 'mp4',
 186             'mp4hd': 'mp4',
 187             'mp4hd2': 'flv',
 188             'mp4hd3': 'flv',
 189             'hd2': 'flv',
 190             'hd3': 'flv',
 191         }
 192         return ext_dict[fm]
 193
 194     def get_format_name(self, fm):
 195         _dict = {
 196             '3gp': 'h6',
 197             '3gphd': 'h5',
 198             'flv': 'h4',
 199             'flvhd': 'h4',
 200             'mp4': 'h3',
 201             'mp4hd': 'h3',
 202             'mp4hd2': 'h4',
 203             'mp4hd3': 'h4',
 204             'hd2': 'h2',
 205             'hd3': 'h1',
 206         }
 207         return _dict[fm]
 208
 209     def _real_extract(self, url):
 210         video_id = self._match_id(url)
 211
 212         self._set_cookie('youku.com', '__ysuid', self.get_ysuid())
 213
 214         def retrieve_data(req_url, note):
 215             headers = {
 216                 'Referer': req_url,
 217             }
 218             self._set_cookie('youku.com', 'xreferrer', 'http://www.youku.com')
 219             req = sanitized_Request(req_url, headers=headers)
 220
 221             cn_verification_proxy = self._downloader.params.get('cn_verification_proxy')
 222             if cn_verification_proxy:
 223                 req.add_header('Ytdl-request-proxy', cn_verification_proxy)
 224
 225             raw_data = self._download_json(req, video_id, note=note)
 226
 227             return raw_data['data']
 228
 229         video_password = self._downloader.params.get('videopassword')
 230
 231         # request basic data
 232         basic_data_url = 'http://play.youku.com/play/get.json?vid=%s&ct=12' % video_id
 233         if video_password:
 234             basic_data_url += '&pwd=%s' % video_password
 235
 236         data = retrieve_data(basic_data_url, 'Downloading JSON metadata')
 237
 238         error = data.get('error')
 239         if error:
 240             error_note = error.get('note')
 241             if error_note is not None and '因版权原因无法观看此视频' in error_note:
 242                 raise ExtractorError(
 243                     'Youku said: Sorry, this video is available in China only', expected=True)
 244             elif error_note and '该视频被设为私密' in error_note:
 245                 raise ExtractorError(
 246                     'Youku said: Sorry, this video is private', expected=True)
 247             else:
 248                 msg = 'Youku server reported error %i' % error.get('code')
 249                 if error_note is not None:
 250                     msg += ': ' + error_note
 251                 raise ExtractorError(msg)
 252
 253         # get video title
 254         title = data['video']['title']
 255
 256         # generate video_urls_dict
 257         video_urls_dict = self.construct_video_urls(data)
 258
 259         # construct info
 260         entries = [{
 261             'id': '%s_part%d' % (video_id, i + 1),
 262             'title': title,
 263             'formats': [],
 264             # some formats are not available for all parts, we have to detect
 265             # which one has all
 266         } for i in range(max(len(v.get('segs')) for v in data['stream']))]
 267         for stream in data['stream']:
 268             if stream.get('channel_type') == 'tail':
 269                 continue
 270             fm = stream.get('stream_type')
 271             video_urls = video_urls_dict[fm]
 272             for video_url, seg, entry in zip(video_urls, stream['segs'], entries):
 273                 entry['formats'].append({
 274                     'url': video_url,
 275                     'format_id': self.get_format_name(fm),
 276                     'ext': self.parse_ext_l(fm),
 277                     'filesize': int(seg['size']),
 278                 })
 279
 280         return {
 281             '_type': 'multi_video',
 282             'id': video_id,
 283             'title': title,
 284             'entries': entries,
 285         }