yt_dlp/extractor/yandexdisk.py

   1 # coding: utf-8
   2 from __future__ import unicode_literals
   3
   4 import json
   5
   6 from .common import InfoExtractor
   7 from ..utils import (
   8     determine_ext,
   9     float_or_none,
  10     int_or_none,
  11     mimetype2ext,
  12     try_get,
  13     urljoin,
  14 )
  15
  16
  17 class YandexDiskIE(InfoExtractor):
  18     _VALID_URL = r'''(?x)https?://
  19         (?P<domain>
  20             yadi\.sk|
  21             disk\.yandex\.
  22                 (?:
  23                     az|
  24                     by|
  25                     co(?:m(?:\.(?:am|ge|tr))?|\.il)|
  26                     ee|
  27                     fr|
  28                     k[gz]|
  29                     l[tv]|
  30                     md|
  31                     t[jm]|
  32                     u[az]|
  33                     ru
  34                 )
  35         )/(?:[di]/|public.*?\bhash=)(?P<id>[^/?#&]+)'''
  36
  37     _TESTS = [{
  38         'url': 'https://yadi.sk/i/VdOeDou8eZs6Y',
  39         'md5': 'a4a8d52958c8fddcf9845935070402ae',
  40         'info_dict': {
  41             'id': 'VdOeDou8eZs6Y',
  42             'ext': 'mp4',
  43             'title': '4.mp4',
  44             'duration': 168.6,
  45             'uploader': 'y.botova',
  46             'uploader_id': '300043621',
  47             'view_count': int,
  48         },
  49         'expected_warnings': ['Unable to download JSON metadata'],
  50     }, {
  51         'url': 'https://yadi.sk/d/h3WAXvDS3Li3Ce',
  52         'only_matching': True,
  53     }, {
  54         'url': 'https://yadi.sk/public?hash=5DZ296JK9GWCLp02f6jrObjnctjRxMs8L6%2B%2FuhNqk38%3D',
  55         'only_matching': True,
  56     }]
  57
  58     def _real_extract(self, url):
  59         domain, video_id = self._match_valid_url(url).groups()
  60
  61         webpage = self._download_webpage(url, video_id)
  62         store = self._parse_json(self._search_regex(
  63             r'<script[^>]+id="store-prefetch"[^>]*>\s*({.+?})\s*</script>',
  64             webpage, 'store'), video_id)
  65         resource = store['resources'][store['rootResourceId']]
  66
  67         title = resource['name']
  68         meta = resource.get('meta') or {}
  69
  70         public_url = meta.get('short_url')
  71         if public_url:
  72             video_id = self._match_id(public_url)
  73
  74         source_url = (self._download_json(
  75             'https://cloud-api.yandex.net/v1/disk/public/resources/download',
  76             video_id, query={'public_key': url}, fatal=False) or {}).get('href')
  77         video_streams = resource.get('videoStreams') or {}
  78         video_hash = resource.get('hash') or url
  79         environment = store.get('environment') or {}
  80         sk = environment.get('sk')
  81         yandexuid = environment.get('yandexuid')
  82         if sk and yandexuid and not (source_url and video_streams):
  83             self._set_cookie(domain, 'yandexuid', yandexuid)
  84
  85             def call_api(action):
  86                 return (self._download_json(
  87                     urljoin(url, '/public/api/') + action, video_id, data=json.dumps({
  88                         'hash': video_hash,
  89                         'sk': sk,
  90                     }).encode(), headers={
  91                         'Content-Type': 'text/plain',
  92                     }, fatal=False) or {}).get('data') or {}
  93             if not source_url:
  94                 # TODO: figure out how to detect if download limit has
  95                 # been reached and then avoid unnecessary source format
  96                 # extraction requests
  97                 source_url = call_api('download-url').get('url')
  98             if not video_streams:
  99                 video_streams = call_api('get-video-streams')
 100
 101         formats = []
 102         if source_url:
 103             formats.append({
 104                 'url': source_url,
 105                 'format_id': 'source',
 106                 'ext': determine_ext(title, meta.get('ext') or mimetype2ext(meta.get('mime_type')) or 'mp4'),
 107                 'quality': 1,
 108                 'filesize': int_or_none(meta.get('size'))
 109             })
 110
 111         for video in (video_streams.get('videos') or []):
 112             format_url = video.get('url')
 113             if not format_url:
 114                 continue
 115             if video.get('dimension') == 'adaptive':
 116                 formats.extend(self._extract_m3u8_formats(
 117                     format_url, video_id, 'mp4', 'm3u8_native',
 118                     m3u8_id='hls', fatal=False))
 119             else:
 120                 size = video.get('size') or {}
 121                 height = int_or_none(size.get('height'))
 122                 format_id = 'hls'
 123                 if height:
 124                     format_id += '-%dp' % height
 125                 formats.append({
 126                     'ext': 'mp4',
 127                     'format_id': format_id,
 128                     'height': height,
 129                     'protocol': 'm3u8_native',
 130                     'url': format_url,
 131                     'width': int_or_none(size.get('width')),
 132                 })
 133         self._sort_formats(formats)
 134
 135         uid = resource.get('uid')
 136         display_name = try_get(store, lambda x: x['users'][uid]['displayName'])
 137
 138         return {
 139             'id': video_id,
 140             'title': title,
 141             'duration': float_or_none(video_streams.get('duration'), 1000),
 142             'uploader': display_name,
 143             'uploader_id': uid,
 144             'view_count': int_or_none(meta.get('views_counter')),
 145             'formats': formats,
 146         }