3 from .common
import InfoExtractor
4 from .vimeo
import VimeoIE
5 from ..compat
import compat_urllib_parse_unquote
6 from ..networking
.exceptions
import HTTPError
23 class PatreonBaseIE(InfoExtractor
):
24 USER_AGENT
= 'Patreon/7.6.28 (Android; Android 11; Scale/2.10)'
26 def _call_api(self
, ep
, item_id
, query
=None, headers
=None, fatal
=True, note
=None):
29 if 'User-Agent' not in headers
:
30 headers
['User-Agent'] = self
.USER_AGENT
32 query
.update({'json-api-version': 1.0}
)
35 return self
._download
_json
(
36 f
'https://www.patreon.com/api/{ep}',
37 item_id
, note
='Downloading API JSON' if not note
else note
,
38 query
=query
, fatal
=fatal
, headers
=headers
)
39 except ExtractorError
as e
:
40 if not isinstance(e
.cause
, HTTPError
) or mimetype2ext(e
.cause
.response
.headers
.get('Content-Type')) != 'json':
42 err_json
= self
._parse
_json
(self
._webpage
_read
_content
(e
.cause
.response
, None, item_id
), item_id
, fatal
=False)
43 err_message
= traverse_obj(err_json
, ('errors', ..., 'detail'), get_all
=False)
45 raise ExtractorError(f
'Patreon said: {err_message}', expected
=True)
49 class PatreonIE(PatreonBaseIE
):
50 _VALID_URL
= r
'https?://(?:www\.)?patreon\.com/(?:creation\?hid=|posts/(?:[\w-]+-)?)(?P<id>\d+)'
52 'url': 'http://www.patreon.com/creation?hid=743933',
53 'md5': 'e25505eec1053a6e6813b8ed369875cc',
57 'title': 'Episode 166: David Smalley of Dogma Debate',
58 'description': 'md5:34d207dd29aa90e24f1b3f58841b81c7',
59 'uploader': 'Cognitive Dissonance Podcast',
60 'thumbnail': 're:^https?://.*$',
61 'timestamp': 1406473987,
62 'upload_date': '20140727',
63 'uploader_id': '87145',
66 'uploader_url': 'https://www.patreon.com/dissonancepod',
67 'channel_id': '80642',
68 'channel_url': 'https://www.patreon.com/dissonancepod',
69 'channel_follower_count': int,
72 'url': 'http://www.patreon.com/creation?hid=754133',
73 'md5': '3eb09345bf44bf60451b8b0b81759d0a',
77 'title': 'CD 167 Extra',
78 'uploader': 'Cognitive Dissonance Podcast',
79 'thumbnail': 're:^https?://.*$',
82 'uploader_url': 'https://www.patreon.com/dissonancepod',
84 'skip': 'Patron-only content',
86 'url': 'https://www.patreon.com/creation?hid=1682498',
90 'title': 'I\'m on Patreon!',
91 'uploader': 'TraciJHines',
92 'thumbnail': 're:^https?://.*$',
93 'upload_date': '20150211',
94 'description': 'md5:8af6425f50bd46fbf29f3db0fc3a8364',
95 'uploader_id': '@TraciHinesMusic',
96 'categories': ['Entertainment'],
101 'channel': 'TraciJHines',
102 'channel_url': 'https://www.youtube.com/channel/UCGLim4T2loE5rwCMdpCIPVg',
103 'live_status': 'not_live',
105 'channel_id': 'UCGLim4T2loE5rwCMdpCIPVg',
106 'availability': 'public',
107 'channel_follower_count': int,
108 'playable_in_embed': True,
109 'uploader_url': 'https://www.youtube.com/@TraciHinesMusic',
110 'comment_count': int,
111 'channel_is_verified': True,
112 'chapters': 'count:4',
116 'skip_download': True,
119 'url': 'https://www.patreon.com/posts/episode-166-of-743933',
120 'only_matching': True,
122 'url': 'https://www.patreon.com/posts/743933',
123 'only_matching': True,
125 'url': 'https://www.patreon.com/posts/kitchen-as-seen-51706779',
126 'md5': '96656690071f6d64895866008484251b',
130 'title': 'KITCHEN AS SEEN ON DEEZ NUTS EXTENDED!',
131 'uploader': 'Cold Ones',
132 'thumbnail': 're:^https?://.*$',
133 'upload_date': '20210526',
134 'description': 'md5:557a409bd79d3898689419094934ba79',
135 'uploader_id': '14936315',
137 'skip': 'Patron-only content'
139 # m3u8 video (https://github.com/yt-dlp/yt-dlp/issues/2277)
140 'url': 'https://www.patreon.com/posts/video-sketchbook-32452882',
144 'comment_count': int,
145 'uploader_id': '4301314',
147 'timestamp': 1576696962,
148 'upload_date': '20191218',
149 'thumbnail': r
're:^https?://.*$',
150 'uploader_url': 'https://www.patreon.com/loish',
151 'description': 'md5:e2693e97ee299c8ece47ffdb67e7d9d2',
152 'title': 'VIDEO // sketchbook flipthrough',
153 'uploader': 'Loish ',
154 'tags': ['sketchbook', 'video'],
155 'channel_id': '1641751',
156 'channel_url': 'https://www.patreon.com/loish',
157 'channel_follower_count': int,
160 # bad videos under media (if media is included). Real one is under post_file
161 'url': 'https://www.patreon.com/posts/premium-access-70282931',
165 'title': '[Premium Access + Uncut] The Office - 2x6 The Fight - Group Reaction',
166 'channel_url': 'https://www.patreon.com/thenormies',
167 'channel_id': '573397',
168 'uploader_id': '2929435',
169 'uploader': 'The Normies',
170 'description': 'md5:79c9fd8778e2cef84049a94c058a5e23',
171 'comment_count': int,
172 'upload_date': '20220809',
173 'thumbnail': r
're:^https?://.*$',
174 'channel_follower_count': int,
176 'timestamp': 1660052820,
177 'tags': ['The Office', 'early access', 'uncut'],
178 'uploader_url': 'https://www.patreon.com/thenormies',
180 'skip': 'Patron-only content',
182 # dead vimeo and embed URLs, need to extract post_file
183 'url': 'https://www.patreon.com/posts/hunter-x-hunter-34007913',
187 'title': 'Hunter x Hunter | Kurapika DESTROYS Uvogin!!!',
189 'uploader': 'YaBoyRoshi',
190 'timestamp': 1581636833,
191 'channel_url': 'https://www.patreon.com/yaboyroshi',
192 'thumbnail': r
're:^https?://.*$',
193 'tags': ['Hunter x Hunter'],
194 'uploader_id': '14264111',
195 'comment_count': int,
196 'channel_follower_count': int,
197 'description': 'Kurapika is a walking cheat code!',
198 'upload_date': '20200213',
199 'channel_id': '2147162',
200 'uploader_url': 'https://www.patreon.com/yaboyroshi',
204 def _real_extract(self
, url
):
205 video_id
= self
._match
_id
(url
)
206 post
= self
._call
_api
(
207 f
'posts/{video_id}', video_id
, query
={
208 'fields[media]': 'download_url,mimetype,size_bytes',
209 'fields[post]': 'comment_count,content,embed,image,like_count,post_file,published_at,title,current_user_can_view',
210 'fields[user]': 'full_name,url',
211 'fields[post_tag]': 'value',
212 'fields[campaign]': 'url,name,patron_count',
213 'json-api-use-default-includes': 'false',
214 'include': 'audio,user,user_defined_tags,campaign,attachments_media',
216 attributes
= post
['data']['attributes']
217 title
= attributes
['title'].strip()
218 image
= attributes
.get('image') or {}
222 'description': clean_html(attributes
.get('content')),
223 'thumbnail': image
.get('large_url') or image
.get('url'),
224 'timestamp': parse_iso8601(attributes
.get('published_at')),
225 'like_count': int_or_none(attributes
.get('like_count')),
226 'comment_count': int_or_none(attributes
.get('comment_count')),
228 can_view_post
= traverse_obj(attributes
, 'current_user_can_view')
229 if can_view_post
and info
['comment_count']:
230 info
['__post_extractor'] = self
.extract_comments(video_id
)
232 for i
in post
.get('included', []):
233 i_type
= i
.get('type')
234 if i_type
== 'media':
235 media_attributes
= i
.get('attributes') or {}
236 download_url
= media_attributes
.get('download_url')
237 ext
= mimetype2ext(media_attributes
.get('mimetype'))
239 # if size_bytes is None, this media file is likely unavailable
240 # See: https://github.com/yt-dlp/yt-dlp/issues/4608
241 size_bytes
= int_or_none(media_attributes
.get('size_bytes'))
242 if download_url
and ext
in KNOWN_EXTENSIONS
and size_bytes
is not None:
243 # XXX: what happens if there are multiple attachments?
247 'filesize': size_bytes
,
250 elif i_type
== 'user':
251 user_attributes
= i
.get('attributes')
254 'uploader': user_attributes
.get('full_name'),
255 'uploader_id': str_or_none(i
.get('id')),
256 'uploader_url': user_attributes
.get('url'),
259 elif i_type
== 'post_tag':
260 info
.setdefault('tags', []).append(traverse_obj(i
, ('attributes', 'value')))
262 elif i_type
== 'campaign':
264 'channel': traverse_obj(i
, ('attributes', 'title')),
265 'channel_id': str_or_none(i
.get('id')),
266 'channel_url': traverse_obj(i
, ('attributes', 'url')),
267 'channel_follower_count': int_or_none(traverse_obj(i
, ('attributes', 'patron_count'))),
270 # handle Vimeo embeds
271 if try_get(attributes
, lambda x
: x
['embed']['provider']) == 'Vimeo':
272 embed_html
= try_get(attributes
, lambda x
: x
['embed']['html'])
273 v_url
= url_or_none(compat_urllib_parse_unquote(
274 self
._search
_regex
(r
'(https(?:%3A%2F%2F|://)player\.vimeo\.com.+app_id(?:=|%3D)+\d+)', embed_html
, 'vimeo url', fatal
=False)))
276 v_url
= VimeoIE
._smuggle
_referrer
(v_url
, 'https://patreon.com')
277 if self
._request
_webpage
(v_url
, video_id
, 'Checking Vimeo embed URL', fatal
=False, errnote
=False):
278 return self
.url_result(v_url
, VimeoIE
, url_transparent
=True, **info
)
280 embed_url
= try_get(attributes
, lambda x
: x
['embed']['url'])
281 if embed_url
and self
._request
_webpage
(embed_url
, video_id
, 'Checking embed URL', fatal
=False, errnote
=False):
282 return self
.url_result(embed_url
, **info
)
284 post_file
= traverse_obj(attributes
, 'post_file')
286 name
= post_file
.get('name')
287 ext
= determine_ext(name
)
288 if ext
in KNOWN_EXTENSIONS
:
292 'url': post_file
['url'],
294 elif name
== 'video' or determine_ext(post_file
.get('url')) == 'm3u8':
295 formats
, subtitles
= self
._extract
_m
3u8_formats
_and
_subtitles
(post_file
['url'], video_id
)
299 'subtitles': subtitles
,
302 if can_view_post
is False:
303 self
.raise_no_formats('You do not have access to this post', video_id
=video_id
, expected
=True)
305 self
.raise_no_formats('No supported media found in this post', video_id
=video_id
, expected
=True)
308 def _get_comments(self
, post_id
):
313 'include': 'parent.commenter.campaign,parent.post.user,parent.post.campaign.creator,parent.replies.parent,parent.replies.commenter.campaign,parent.replies.post.user,parent.replies.post.campaign.creator,commenter.campaign,post.user,post.campaign.creator,replies.parent,replies.commenter.campaign,replies.post.user,replies.post.campaign.creator,on_behalf_of_campaign',
314 'fields[comment]': 'body,created,is_by_creator',
315 'fields[user]': 'image_url,full_name,url',
316 'filter[flair]': 'image_tiny_url,name',
318 'json-api-version': 1.0,
319 'json-api-use-default-includes': 'false',
322 for page
in itertools
.count(1):
324 params
.update({'page[cursor]': cursor}
if cursor
else {})
325 response
= self
._call
_api
(
326 f
'posts/{post_id}/comments', post_id
, query
=params
, note
='Downloading comments page %d' % page
)
329 for comment
in traverse_obj(response
, (('data', ('included', lambda _
, v
: v
['type'] == 'comment')), ...)):
331 comment_id
= comment
.get('id')
332 attributes
= comment
.get('attributes') or {}
333 if comment_id
is None:
335 author_id
= traverse_obj(comment
, ('relationships', 'commenter', 'data', 'id'))
336 author_info
= traverse_obj(
337 response
, ('included', lambda _
, v
: v
['id'] == author_id
and v
['type'] == 'user', 'attributes'),
338 get_all
=False, expected_type
=dict, default
={})
342 'text': attributes
.get('body'),
343 'timestamp': parse_iso8601(attributes
.get('created')),
344 'parent': traverse_obj(comment
, ('relationships', 'parent', 'data', 'id'), default
='root'),
345 'author_is_uploader': attributes
.get('is_by_creator'),
346 'author_id': author_id
,
347 'author': author_info
.get('full_name'),
348 'author_thumbnail': author_info
.get('image_url'),
351 if count
< traverse_obj(response
, ('meta', 'count')):
352 cursor
= traverse_obj(response
, ('data', -1, 'id'))
358 class PatreonCampaignIE(PatreonBaseIE
):
360 _VALID_URL
= r
'https?://(?:www\.)?patreon\.com/(?!rss)(?:(?:m/(?P<campaign_id>\d+))|(?P<vanity>[-\w]+))'
362 'url': 'https://www.patreon.com/dissonancepod/',
364 'title': 'Cognitive Dissonance Podcast',
365 'channel_url': 'https://www.patreon.com/dissonancepod',
367 'description': 'md5:eb2fa8b83da7ab887adeac34da6b7af7',
368 'channel_id': '80642',
369 'channel': 'Cognitive Dissonance Podcast',
371 'channel_follower_count': int,
372 'uploader_id': '87145',
373 'uploader_url': 'https://www.patreon.com/dissonancepod',
374 'uploader': 'Cognitive Dissonance Podcast',
375 'thumbnail': r
're:^https?://.*$',
377 'playlist_mincount': 68,
379 'url': 'https://www.patreon.com/m/4767637/posts',
381 'title': 'Not Just Bikes',
382 'channel_follower_count': int,
384 'channel_id': '4767637',
385 'channel_url': 'https://www.patreon.com/notjustbikes',
386 'description': 'md5:595c6e7dca76ae615b1d38c298a287a1',
388 'channel': 'Not Just Bikes',
389 'uploader_url': 'https://www.patreon.com/notjustbikes',
390 'uploader': 'Not Just Bikes',
391 'uploader_id': '37306634',
392 'thumbnail': r
're:^https?://.*$',
394 'playlist_mincount': 71
396 'url': 'https://www.patreon.com/dissonancepod/posts',
397 'only_matching': True
399 'url': 'https://www.patreon.com/m/5932659',
400 'only_matching': True
404 def suitable(cls
, url
):
405 return False if PatreonIE
.suitable(url
) else super(PatreonCampaignIE
, cls
).suitable(url
)
407 def _entries(self
, campaign_id
):
410 'fields[post]': 'patreon_url,url',
411 'filter[campaign_id]': campaign_id
,
412 'filter[is_draft]': 'false',
413 'sort': '-published_at',
414 'json-api-use-default-includes': 'false',
417 for page
in itertools
.count(1):
419 params
.update({'page[cursor]': cursor}
if cursor
else {})
420 posts_json
= self
._call
_api
('posts', campaign_id
, query
=params
, note
='Downloading posts page %d' % page
)
422 cursor
= traverse_obj(posts_json
, ('meta', 'pagination', 'cursors', 'next'))
423 for post_url
in traverse_obj(posts_json
, ('data', ..., 'attributes', 'patreon_url')):
424 yield self
.url_result(urljoin('https://www.patreon.com/', post_url
), PatreonIE
)
429 def _real_extract(self
, url
):
431 campaign_id
, vanity
= self
._match
_valid
_url
(url
).group('campaign_id', 'vanity')
432 if campaign_id
is None:
433 webpage
= self
._download
_webpage
(url
, vanity
, headers
={'User-Agent': self.USER_AGENT}
)
434 campaign_id
= self
._search
_regex
(r
'https://www.patreon.com/api/campaigns/(\d+)/?', webpage
, 'Campaign ID')
437 'json-api-use-default-includes': 'false',
438 'fields[user]': 'full_name,url',
439 'fields[campaign]': 'name,summary,url,patron_count,creation_count,is_nsfw,avatar_photo_url',
443 campaign_response
= self
._call
_api
(
444 f
'campaigns/{campaign_id}', campaign_id
,
445 note
='Downloading campaign info', fatal
=False,
448 campaign_info
= campaign_response
.get('data') or {}
449 channel_name
= traverse_obj(campaign_info
, ('attributes', 'name'))
450 user_info
= traverse_obj(
451 campaign_response
, ('included', lambda _
, v
: v
['type'] == 'user'),
452 default
={}, expected_type
=dict, get_all
=False)
457 'title': channel_name
,
458 'entries': self
._entries
(campaign_id
),
459 'description': clean_html(traverse_obj(campaign_info
, ('attributes', 'summary'))),
460 'channel_url': traverse_obj(campaign_info
, ('attributes', 'url')),
461 'channel_follower_count': int_or_none(traverse_obj(campaign_info
, ('attributes', 'patron_count'))),
462 'channel_id': campaign_id
,
463 'channel': channel_name
,
464 'uploader_url': traverse_obj(user_info
, ('attributes', 'url')),
465 'uploader_id': str_or_none(user_info
.get('id')),
466 'uploader': traverse_obj(user_info
, ('attributes', 'full_name')),
467 'playlist_count': traverse_obj(campaign_info
, ('attributes', 'creation_count')),
468 'age_limit': 18 if traverse_obj(campaign_info
, ('attributes', 'is_nsfw')) else 0,
469 'thumbnail': url_or_none(traverse_obj(campaign_info
, ('attributes', 'avatar_photo_url'))),