2 from urllib
.error
import HTTPError
4 from .common
import InfoExtractor
5 from .vimeo
import VimeoIE
7 from ..compat
import compat_urllib_parse_unquote
23 class PatreonBaseIE(InfoExtractor
):
24 USER_AGENT
= 'Patreon/7.6.28 (Android; Android 11; Scale/2.10)'
26 def _call_api(self
, ep
, item_id
, query
=None, headers
=None, fatal
=True, note
=None):
29 if 'User-Agent' not in headers
:
30 headers
['User-Agent'] = self
.USER_AGENT
32 query
.update({'json-api-version': 1.0}
)
35 return self
._download
_json
(
36 f
'https://www.patreon.com/api/{ep}',
37 item_id
, note
='Downloading API JSON' if not note
else note
,
38 query
=query
, fatal
=fatal
, headers
=headers
)
39 except ExtractorError
as e
:
40 if not isinstance(e
.cause
, HTTPError
) or mimetype2ext(e
.cause
.headers
.get('Content-Type')) != 'json':
42 err_json
= self
._parse
_json
(self
._webpage
_read
_content
(e
.cause
, None, item_id
), item_id
, fatal
=False)
43 err_message
= traverse_obj(err_json
, ('errors', ..., 'detail'), get_all
=False)
45 raise ExtractorError(f
'Patreon said: {err_message}', expected
=True)
49 class PatreonIE(PatreonBaseIE
):
50 _VALID_URL
= r
'https?://(?:www\.)?patreon\.com/(?:creation\?hid=|posts/(?:[\w-]+-)?)(?P<id>\d+)'
52 'url': 'http://www.patreon.com/creation?hid=743933',
53 'md5': 'e25505eec1053a6e6813b8ed369875cc',
57 'title': 'Episode 166: David Smalley of Dogma Debate',
58 'description': 'md5:34d207dd29aa90e24f1b3f58841b81c7',
59 'uploader': 'Cognitive Dissonance Podcast',
60 'thumbnail': 're:^https?://.*$',
61 'timestamp': 1406473987,
62 'upload_date': '20140727',
63 'uploader_id': '87145',
66 'uploader_url': 'https://www.patreon.com/dissonancepod',
67 'channel_id': '80642',
68 'channel_url': 'https://www.patreon.com/dissonancepod',
69 'channel_follower_count': int,
72 'url': 'http://www.patreon.com/creation?hid=754133',
73 'md5': '3eb09345bf44bf60451b8b0b81759d0a',
77 'title': 'CD 167 Extra',
78 'uploader': 'Cognitive Dissonance Podcast',
79 'thumbnail': 're:^https?://.*$',
82 'uploader_url': 'https://www.patreon.com/dissonancepod',
84 'skip': 'Patron-only content',
86 'url': 'https://www.patreon.com/creation?hid=1682498',
90 'title': 'I\'m on Patreon!',
91 'uploader': 'TraciJHines',
92 'thumbnail': 're:^https?://.*$',
93 'upload_date': '20150211',
94 'description': 'md5:8af6425f50bd46fbf29f3db0fc3a8364',
95 'uploader_id': 'TraciJHines',
96 'categories': ['Entertainment'],
101 'channel': 'TraciJHines',
102 'channel_url': 'https://www.youtube.com/channel/UCGLim4T2loE5rwCMdpCIPVg',
103 'live_status': 'not_live',
105 'channel_id': 'UCGLim4T2loE5rwCMdpCIPVg',
106 'availability': 'public',
107 'channel_follower_count': int,
108 'playable_in_embed': True,
109 'uploader_url': 'http://www.youtube.com/user/TraciJHines',
110 'comment_count': int,
114 'skip_download': True,
117 'url': 'https://www.patreon.com/posts/episode-166-of-743933',
118 'only_matching': True,
120 'url': 'https://www.patreon.com/posts/743933',
121 'only_matching': True,
123 'url': 'https://www.patreon.com/posts/kitchen-as-seen-51706779',
124 'md5': '96656690071f6d64895866008484251b',
128 'title': 'KITCHEN AS SEEN ON DEEZ NUTS EXTENDED!',
129 'uploader': 'Cold Ones',
130 'thumbnail': 're:^https?://.*$',
131 'upload_date': '20210526',
132 'description': 'md5:557a409bd79d3898689419094934ba79',
133 'uploader_id': '14936315',
135 'skip': 'Patron-only content'
137 # m3u8 video (https://github.com/yt-dlp/yt-dlp/issues/2277)
138 'url': 'https://www.patreon.com/posts/video-sketchbook-32452882',
142 'comment_count': int,
143 'uploader_id': '4301314',
145 'timestamp': 1576696962,
146 'upload_date': '20191218',
147 'thumbnail': r
're:^https?://.*$',
148 'uploader_url': 'https://www.patreon.com/loish',
149 'description': 'md5:e2693e97ee299c8ece47ffdb67e7d9d2',
150 'title': 'VIDEO // sketchbook flipthrough',
151 'uploader': 'Loish ',
152 'tags': ['sketchbook', 'video'],
153 'channel_id': '1641751',
154 'channel_url': 'https://www.patreon.com/loish',
155 'channel_follower_count': int,
159 def _real_extract(self
, url
):
160 video_id
= self
._match
_id
(url
)
161 post
= self
._call
_api
(
162 f
'posts/{video_id}', video_id
, query
={
163 'fields[media]': 'download_url,mimetype,size_bytes',
164 'fields[post]': 'comment_count,content,embed,image,like_count,post_file,published_at,title,current_user_can_view',
165 'fields[user]': 'full_name,url',
166 'fields[post_tag]': 'value',
167 'fields[campaign]': 'url,name,patron_count',
168 'json-api-use-default-includes': 'false',
169 'include': 'media,user,user_defined_tags,campaign',
171 attributes
= post
['data']['attributes']
172 title
= attributes
['title'].strip()
173 image
= attributes
.get('image') or {}
177 'description': clean_html(attributes
.get('content')),
178 'thumbnail': image
.get('large_url') or image
.get('url'),
179 'timestamp': parse_iso8601(attributes
.get('published_at')),
180 'like_count': int_or_none(attributes
.get('like_count')),
181 'comment_count': int_or_none(attributes
.get('comment_count')),
183 can_view_post
= traverse_obj(attributes
, 'current_user_can_view')
184 if can_view_post
and info
['comment_count']:
185 info
['__post_extractor'] = self
.extract_comments(video_id
)
187 for i
in post
.get('included', []):
188 i_type
= i
.get('type')
189 if i_type
== 'media':
190 media_attributes
= i
.get('attributes') or {}
191 download_url
= media_attributes
.get('download_url')
192 ext
= mimetype2ext(media_attributes
.get('mimetype'))
193 if download_url
and ext
in KNOWN_EXTENSIONS
:
197 'filesize': int_or_none(media_attributes
.get('size_bytes')),
200 elif i_type
== 'user':
201 user_attributes
= i
.get('attributes')
204 'uploader': user_attributes
.get('full_name'),
205 'uploader_id': str_or_none(i
.get('id')),
206 'uploader_url': user_attributes
.get('url'),
209 elif i_type
== 'post_tag':
210 info
.setdefault('tags', []).append(traverse_obj(i
, ('attributes', 'value')))
212 elif i_type
== 'campaign':
214 'channel': traverse_obj(i
, ('attributes', 'title')),
215 'channel_id': str_or_none(i
.get('id')),
216 'channel_url': traverse_obj(i
, ('attributes', 'url')),
217 'channel_follower_count': int_or_none(traverse_obj(i
, ('attributes', 'patron_count'))),
220 # handle Vimeo embeds
221 if try_get(attributes
, lambda x
: x
['embed']['provider']) == 'Vimeo':
222 embed_html
= try_get(attributes
, lambda x
: x
['embed']['html'])
223 v_url
= url_or_none(compat_urllib_parse_unquote(
224 self
._search
_regex
(r
'(https(?:%3A%2F%2F|://)player\.vimeo\.com.+app_id(?:=|%3D)+\d+)', embed_html
, 'vimeo url', fatal
=False)))
228 '_type': 'url_transparent',
229 'url': VimeoIE
._smuggle
_referrer
(v_url
, 'https://patreon.com'),
233 embed_url
= try_get(attributes
, lambda x
: x
['embed']['url'])
241 post_file
= traverse_obj(attributes
, 'post_file')
243 name
= post_file
.get('name')
244 ext
= determine_ext(name
)
245 if ext
in KNOWN_EXTENSIONS
:
249 'url': post_file
['url'],
251 elif name
== 'video':
252 formats
, subtitles
= self
._extract
_m
3u8_formats
_and
_subtitles
(post_file
['url'], video_id
)
256 'subtitles': subtitles
,
259 if can_view_post
is False:
260 self
.raise_no_formats('You do not have access to this post', video_id
=video_id
, expected
=True)
262 self
.raise_no_formats('No supported media found in this post', video_id
=video_id
, expected
=True)
265 def _get_comments(self
, post_id
):
270 'include': 'parent.commenter.campaign,parent.post.user,parent.post.campaign.creator,parent.replies.parent,parent.replies.commenter.campaign,parent.replies.post.user,parent.replies.post.campaign.creator,commenter.campaign,post.user,post.campaign.creator,replies.parent,replies.commenter.campaign,replies.post.user,replies.post.campaign.creator,on_behalf_of_campaign',
271 'fields[comment]': 'body,created,is_by_creator',
272 'fields[user]': 'image_url,full_name,url',
273 'filter[flair]': 'image_tiny_url,name',
275 'json-api-version': 1.0,
276 'json-api-use-default-includes': 'false',
279 for page
in itertools
.count(1):
281 params
.update({'page[cursor]': cursor}
if cursor
else {})
282 response
= self
._call
_api
(
283 f
'posts/{post_id}/comments', post_id
, query
=params
, note
='Downloading comments page %d' % page
)
286 for comment
in traverse_obj(response
, (('data', ('included', lambda _
, v
: v
['type'] == 'comment')), ...), default
=[]):
288 comment_id
= comment
.get('id')
289 attributes
= comment
.get('attributes') or {}
290 if comment_id
is None:
292 author_id
= traverse_obj(comment
, ('relationships', 'commenter', 'data', 'id'))
293 author_info
= traverse_obj(
294 response
, ('included', lambda _
, v
: v
['id'] == author_id
and v
['type'] == 'user', 'attributes'),
295 get_all
=False, expected_type
=dict, default
={})
299 'text': attributes
.get('body'),
300 'timestamp': parse_iso8601(attributes
.get('created')),
301 'parent': traverse_obj(comment
, ('relationships', 'parent', 'data', 'id'), default
='root'),
302 'author_is_uploader': attributes
.get('is_by_creator'),
303 'author_id': author_id
,
304 'author': author_info
.get('full_name'),
305 'author_thumbnail': author_info
.get('image_url'),
308 if count
< traverse_obj(response
, ('meta', 'count')):
309 cursor
= traverse_obj(response
, ('data', -1, 'id'))
315 class PatreonCampaignIE(PatreonBaseIE
):
317 _VALID_URL
= r
'https?://(?:www\.)?patreon\.com/(?!rss)(?:(?:m/(?P<campaign_id>\d+))|(?P<vanity>[-\w]+))'
319 'url': 'https://www.patreon.com/dissonancepod/',
321 'title': 'Cognitive Dissonance Podcast',
322 'channel_url': 'https://www.patreon.com/dissonancepod',
324 'description': 'md5:eb2fa8b83da7ab887adeac34da6b7af7',
325 'channel_id': '80642',
326 'channel': 'Cognitive Dissonance Podcast',
328 'channel_follower_count': int,
329 'uploader_id': '87145',
330 'uploader_url': 'https://www.patreon.com/dissonancepod',
331 'uploader': 'Cognitive Dissonance Podcast',
332 'thumbnail': r
're:^https?://.*$',
334 'playlist_mincount': 68,
336 'url': 'https://www.patreon.com/m/4767637/posts',
338 'title': 'Not Just Bikes',
339 'channel_follower_count': int,
341 'channel_id': '4767637',
342 'channel_url': 'https://www.patreon.com/notjustbikes',
343 'description': 'md5:595c6e7dca76ae615b1d38c298a287a1',
345 'channel': 'Not Just Bikes',
346 'uploader_url': 'https://www.patreon.com/notjustbikes',
347 'uploader': 'Not Just Bikes',
348 'uploader_id': '37306634',
349 'thumbnail': r
're:^https?://.*$',
351 'playlist_mincount': 71
353 'url': 'https://www.patreon.com/dissonancepod/posts',
354 'only_matching': True
356 'url': 'https://www.patreon.com/m/5932659',
357 'only_matching': True
361 def suitable(cls
, url
):
362 return False if PatreonIE
.suitable(url
) else super(PatreonCampaignIE
, cls
).suitable(url
)
364 def _entries(self
, campaign_id
):
367 'fields[post]': 'patreon_url,url',
368 'filter[campaign_id]': campaign_id
,
369 'filter[is_draft]': 'false',
370 'sort': '-published_at',
371 'json-api-use-default-includes': 'false',
374 for page
in itertools
.count(1):
376 params
.update({'page[cursor]': cursor}
if cursor
else {})
377 posts_json
= self
._call
_api
('posts', campaign_id
, query
=params
, note
='Downloading posts page %d' % page
)
379 cursor
= traverse_obj(posts_json
, ('meta', 'pagination', 'cursors', 'next'))
380 for post
in posts_json
.get('data') or []:
381 yield self
.url_result(url_or_none(traverse_obj(post
, ('attributes', 'patreon_url'))), 'Patreon')
386 def _real_extract(self
, url
):
388 campaign_id
, vanity
= self
._match
_valid
_url
(url
).group('campaign_id', 'vanity')
389 if campaign_id
is None:
390 webpage
= self
._download
_webpage
(url
, vanity
, headers
={'User-Agent': self.USER_AGENT}
)
391 campaign_id
= self
._search
_regex
(r
'https://www.patreon.com/api/campaigns/(\d+)/?', webpage
, 'Campaign ID')
394 'json-api-use-default-includes': 'false',
395 'fields[user]': 'full_name,url',
396 'fields[campaign]': 'name,summary,url,patron_count,creation_count,is_nsfw,avatar_photo_url',
400 campaign_response
= self
._call
_api
(
401 f
'campaigns/{campaign_id}', campaign_id
,
402 note
='Downloading campaign info', fatal
=False,
405 campaign_info
= campaign_response
.get('data') or {}
406 channel_name
= traverse_obj(campaign_info
, ('attributes', 'name'))
407 user_info
= traverse_obj(
408 campaign_response
, ('included', lambda _
, v
: v
['type'] == 'user'),
409 default
={}, expected_type
=dict, get_all
=False)
414 'title': channel_name
,
415 'entries': self
._entries
(campaign_id
),
416 'description': clean_html(traverse_obj(campaign_info
, ('attributes', 'summary'))),
417 'channel_url': traverse_obj(campaign_info
, ('attributes', 'url')),
418 'channel_follower_count': int_or_none(traverse_obj(campaign_info
, ('attributes', 'patron_count'))),
419 'channel_id': campaign_id
,
420 'channel': channel_name
,
421 'uploader_url': traverse_obj(user_info
, ('attributes', 'url')),
422 'uploader_id': str_or_none(user_info
.get('id')),
423 'uploader': traverse_obj(user_info
, ('attributes', 'full_name')),
424 'playlist_count': traverse_obj(campaign_info
, ('attributes', 'creation_count')),
425 'age_limit': 18 if traverse_obj(campaign_info
, ('attributes', 'is_nsfw')) else 0,
426 'thumbnail': url_or_none(traverse_obj(campaign_info
, ('attributes', 'avatar_photo_url'))),