3 from .common
import InfoExtractor
4 from ..compat
import compat_parse_qs
10 get_element_html_by_id
,
18 class GoogleDriveIE(InfoExtractor
):
22 (?:docs|drive|drive\.usercontent)\.google\.com/
24 (?:uc|open|download)\?.*?id=|
27 video\.google\.com/get_player\?.*?docid=
29 (?P<id>[a-zA-Z0-9_-]{28,})
32 'url': 'https://drive.google.com/file/d/0ByeS4oOUV-49Zzh4R1J6R09zazQ/edit?pli=1',
33 'md5': '5c602afbbf2c1db91831f5d82f678554',
35 'id': '0ByeS4oOUV-49Zzh4R1J6R09zazQ',
37 'title': 'Big Buck Bunny.mp4',
39 'thumbnail': 'https://drive.google.com/thumbnail?id=0ByeS4oOUV-49Zzh4R1J6R09zazQ',
42 # video can't be watched anonymously due to view count limit reached,
43 # but can be downloaded (see https://github.com/ytdl-org/youtube-dl/issues/14046)
44 'url': 'https://drive.google.com/file/d/0B-vUyvmDLdWDcEt4WjBqcmI2XzQ/view',
45 'only_matching': True,
47 # video id is longer than 28 characters
48 'url': 'https://drive.google.com/file/d/1ENcQ_jeCuj7y19s66_Ou9dRP4GKGsodiDQ/edit',
49 'only_matching': True,
51 'url': 'https://drive.google.com/open?id=0B2fjwgkl1A_CX083Tkowdmt6d28',
52 'only_matching': True,
54 'url': 'https://drive.google.com/uc?id=0B2fjwgkl1A_CX083Tkowdmt6d28',
55 'only_matching': True,
57 'url': 'https://drive.usercontent.google.com/download?id=0ByeS4oOUV-49Zzh4R1J6R09zazQ',
58 'only_matching': True,
78 _BASE_URL_CAPTIONS
= 'https://drive.google.com/timedtext'
79 _CAPTIONS_ENTRY_TAG
= {
81 'automatic_captions': 'target',
83 _caption_formats_ext
= []
87 def _extract_embed_urls(cls
, url
, webpage
):
89 r
'<iframe[^>]+src="https?://(?:video\.google\.com/get_player\?.*?docid=|(?:docs|drive)\.google\.com/file/d/)(?P<id>[a-zA-Z0-9_-]{28,})',
92 yield 'https://drive.google.com/file/d/%s' % mobj
.group('id')
94 def _download_subtitles_xml(self
, video_id
, subtitles_id
, hl
):
95 if self
._captions
_xml
:
97 self
._captions
_xml
= self
._download
_xml
(
98 self
._BASE
_URL
_CAPTIONS
, video_id
, query
={
107 }, note
='Downloading subtitles XML',
108 errnote
='Unable to download subtitles XML', fatal
=False)
109 if self
._captions
_xml
:
110 for f
in self
._captions
_xml
.findall('format'):
111 if f
.attrib
.get('fmt_code') and not f
.attrib
.get('default'):
112 self
._caption
_formats
_ext
.append(f
.attrib
['fmt_code'])
114 def _get_captions_by_type(self
, video_id
, subtitles_id
, caption_type
,
115 origin_lang_code
=None):
116 if not subtitles_id
or not caption_type
:
119 for caption_entry
in self
._captions
_xml
.findall(
120 self
._CAPTIONS
_ENTRY
_TAG
[caption_type
]):
121 caption_lang_code
= caption_entry
.attrib
.get('lang_code')
122 if not caption_lang_code
:
124 caption_format_data
= []
125 for caption_format
in self
._caption
_formats
_ext
:
129 'fmt': caption_format
,
130 'lang': (caption_lang_code
if origin_lang_code
is None
131 else origin_lang_code
),
136 if origin_lang_code
is not None:
137 query
.update({'tlang': caption_lang_code}
)
138 caption_format_data
.append({
139 'url': update_url_query(self
._BASE
_URL
_CAPTIONS
, query
),
140 'ext': caption_format
,
142 captions
[caption_lang_code
] = caption_format_data
145 def _get_subtitles(self
, video_id
, subtitles_id
, hl
):
146 if not subtitles_id
or not hl
:
148 self
._download
_subtitles
_xml
(video_id
, subtitles_id
, hl
)
149 if not self
._captions
_xml
:
151 return self
._get
_captions
_by
_type
(video_id
, subtitles_id
, 'subtitles')
153 def _get_automatic_captions(self
, video_id
, subtitles_id
, hl
):
154 if not subtitles_id
or not hl
:
156 self
._download
_subtitles
_xml
(video_id
, subtitles_id
, hl
)
157 if not self
._captions
_xml
:
159 track
= self
._captions
_xml
.find('track')
162 origin_lang_code
= track
.attrib
.get('lang_code')
163 if not origin_lang_code
:
165 return self
._get
_captions
_by
_type
(
166 video_id
, subtitles_id
, 'automatic_captions', origin_lang_code
)
168 def _real_extract(self
, url
):
169 video_id
= self
._match
_id
(url
)
170 video_info
= compat_parse_qs(self
._download
_webpage
(
171 'https://drive.google.com/get_video_info',
172 video_id
, 'Downloading video webpage', query
={'docid': video_id}
))
175 return try_get(video_info
, lambda x
: x
[key
][0])
177 reason
= get_value('reason')
178 title
= get_value('title')
181 fmt_stream_map
= (get_value('fmt_stream_map') or '').split(',')
182 fmt_list
= (get_value('fmt_list') or '').split(',')
183 if fmt_stream_map
and fmt_list
:
187 r
'^(?P<format_id>\d+)/(?P<width>\d+)[xX](?P<height>\d+)', fmt
)
189 resolutions
[mobj
.group('format_id')] = (
190 int(mobj
.group('width')), int(mobj
.group('height')))
192 for fmt_stream
in fmt_stream_map
:
193 fmt_stream_split
= fmt_stream
.split('|')
194 if len(fmt_stream_split
) < 2:
196 format_id
, format_url
= fmt_stream_split
[:2]
198 'url': lowercase_escape(format_url
),
199 'format_id': format_id
,
200 'ext': self
._FORMATS
_EXT
[format_id
],
202 resolution
= resolutions
.get(format_id
)
205 'width': resolution
[0],
206 'height': resolution
[1],
210 source_url
= update_url_query(
211 'https://drive.usercontent.google.com/download', {
213 'export': 'download',
217 def request_source_file(source_url
, kind
, data
=None):
218 return self
._request
_webpage
(
219 source_url
, video_id
, note
='Requesting %s file' % kind
,
220 errnote
='Unable to request %s file' % kind
, fatal
=False, data
=data
)
221 urlh
= request_source_file(source_url
, 'source')
223 def add_source_format(urlh
):
226 title
= self
._search
_regex
(
227 r
'\bfilename="([^"]+)"', urlh
.headers
.get('Content-Disposition'),
228 'title', default
=None)
230 # Use redirect URLs as download URLs in order to calculate
231 # correct cookies in _calc_cookies.
232 # Using original URLs may result in redirect loop due to
233 # google.com's cookies mistakenly used for googleusercontent.com
234 # redirect URLs (see #23919).
236 'ext': determine_ext(title
, 'mp4').lower(),
237 'format_id': 'source',
240 if urlh
.headers
.get('Content-Disposition'):
241 add_source_format(urlh
)
243 confirmation_webpage
= self
._webpage
_read
_content
(
244 urlh
, url
, video_id
, note
='Downloading confirmation page',
245 errnote
='Unable to confirm download', fatal
=False)
246 if confirmation_webpage
:
247 confirmed_source_url
= extract_attributes(
248 get_element_html_by_id('download-form', confirmation_webpage
) or '').get('action')
249 if confirmed_source_url
:
250 urlh
= request_source_file(confirmed_source_url
, 'confirmed source', data
=b
'')
251 if urlh
and urlh
.headers
.get('Content-Disposition'):
252 add_source_format(urlh
)
255 get_element_by_class('uc-error-subcaption', confirmation_webpage
)
256 or get_element_by_class('uc-error-caption', confirmation_webpage
)
257 or 'unable to extract confirmation code')
259 if not formats
and reason
:
261 self
.raise_no_formats(reason
, expected
=True)
263 raise ExtractorError(reason
, expected
=True)
267 ttsurl
= get_value('ttsurl')
269 # the video Id for subtitles will be the last value in the ttsurl
271 subtitles_id
= ttsurl
.encode('utf-8').decode(
272 'unicode_escape').split('=')[-1]
274 self
.cookiejar
.clear(domain
='.google.com', path
='/', name
='NID')
279 'thumbnail': 'https://drive.google.com/thumbnail?id=' + video_id
,
280 'duration': int_or_none(get_value('length_seconds')),
282 'subtitles': self
.extract_subtitles(video_id
, subtitles_id
, hl
),
283 'automatic_captions': self
.extract_automatic_captions(
284 video_id
, subtitles_id
, hl
),
288 class GoogleDriveFolderIE(InfoExtractor
):
289 IE_NAME
= 'GoogleDrive:Folder'
290 _VALID_URL
= r
'https?://(?:docs|drive)\.google\.com/drive/folders/(?P<id>[\w-]{28,})'
292 'url': 'https://drive.google.com/drive/folders/1dQ4sx0-__Nvg65rxTSgQrl7VyW_FZ9QI',
294 'id': '1dQ4sx0-__Nvg65rxTSgQrl7VyW_FZ9QI',
299 _BOUNDARY
= '=====vc17a3rwnndj====='
300 _REQUEST
= "/drive/v2beta/files?openDrive=true&reason=102&syncType=0&errorRecovery=false&q=trashed%20%3D%20false%20and%20'{folder_id}'%20in%20parents&fields=kind%2CnextPageToken%2Citems(kind%2CmodifiedDate%2CmodifiedByMeDate%2ClastViewedByMeDate%2CfileSize%2Cowners(kind%2CpermissionId%2Cid)%2ClastModifyingUser(kind%2CpermissionId%2Cid)%2ChasThumbnail%2CthumbnailVersion%2Ctitle%2Cid%2CresourceKey%2Cshared%2CsharedWithMeDate%2CuserPermission(role)%2CexplicitlyTrashed%2CmimeType%2CquotaBytesUsed%2Ccopyable%2CfileExtension%2CsharingUser(kind%2CpermissionId%2Cid)%2Cspaces%2Cversion%2CteamDriveId%2ChasAugmentedPermissions%2CcreatedDate%2CtrashingUser(kind%2CpermissionId%2Cid)%2CtrashedDate%2Cparents(id)%2CshortcutDetails(targetId%2CtargetMimeType%2CtargetLookupStatus)%2Ccapabilities(canCopy%2CcanDownload%2CcanEdit%2CcanAddChildren%2CcanDelete%2CcanRemoveChildren%2CcanShare%2CcanTrash%2CcanRename%2CcanReadTeamDrive%2CcanMoveTeamDriveItem)%2Clabels(starred%2Ctrashed%2Crestricted%2Cviewed))%2CincompleteSearch&appDataFilter=NO_APP_DATA&spaces=drive&pageToken={page_token}&maxResults=50&supportsTeamDrives=true&includeItemsFromAllDrives=true&corpora=default&orderBy=folder%2Ctitle_natural%20asc&retryCount=0&key={key} HTTP/1.1"
301 _DATA
= f
'''--{_BOUNDARY}
302 content-type: application/http
303 content-transfer-encoding: binary
310 def _call_api(self
, folder_id
, key
, data
, **kwargs
):
311 response
= self
._download
_webpage
(
312 'https://clients6.google.com/batch/drive/v2beta',
313 folder_id
, data
=data
.encode('utf-8'),
315 'Content-Type': 'text/plain;charset=UTF-8;',
316 'Origin': 'https://drive.google.com',
318 '$ct': f
'multipart/mixed; boundary="{self._BOUNDARY}"',
321 return self
._search
_json
('', response
, 'api response', folder_id
, **kwargs
) or {}
323 def _get_folder_items(self
, folder_id
, key
):
325 while page_token
is not None:
326 request
= self
._REQUEST
.format(folder_id
=folder_id
, page_token
=page_token
, key
=key
)
327 page
= self
._call
_api
(folder_id
, key
, self
._DATA
% request
)
328 yield from page
['items']
329 page_token
= page
.get('nextPageToken')
331 def _real_extract(self
, url
):
332 folder_id
= self
._match
_id
(url
)
334 webpage
= self
._download
_webpage
(url
, folder_id
)
335 key
= self
._search
_regex
(r
'"(\w{39})"', webpage
, 'key')
337 folder_info
= self
._call
_api
(folder_id
, key
, self
._DATA
% f
'/drive/v2beta/files/{folder_id} HTTP/1.1', fatal
=False)
339 return self
.playlist_from_matches(
340 self
._get
_folder
_items
(folder_id
, key
), folder_id
, folder_info
.get('title'),
341 ie
=GoogleDriveIE
, getter
=lambda item
: f
'https://drive.google.com/file/d/{item["id"]}')