]>
Commit | Line | Data |
---|---|---|
612e31f5 E |
1 | import re |
2 | import urllib.parse | |
3 | ||
4 | from .common import InfoExtractor | |
12037d8b | 5 | from ..utils import js_to_json, str_or_none, traverse_obj |
612e31f5 E |
6 | |
7 | ||
8 | class SubstackIE(InfoExtractor): | |
9 | _VALID_URL = r'https?://(?P<username>[\w-]+)\.substack\.com/p/(?P<id>[\w-]+)' | |
10 | _TESTS = [{ | |
11 | 'url': 'https://haleynahman.substack.com/p/i-made-a-vlog?s=r', | |
12 | 'md5': 'f27e4fc6252001d48d479f45e65cdfd5', | |
13 | 'info_dict': { | |
14 | 'id': '47660949', | |
15 | 'ext': 'mp4', | |
16 | 'title': 'I MADE A VLOG', | |
12037d8b | 17 | 'description': 'md5:9248af9a759321e1027226f988f54d96', |
612e31f5 E |
18 | 'thumbnail': 'md5:bec758a34d8ee9142d43bcebdf33af18', |
19 | 'uploader': 'Maybe Baby', | |
20 | 'uploader_id': '33628', | |
add96eb9 | 21 | }, |
612e31f5 E |
22 | }, { |
23 | 'url': 'https://haleynahman.substack.com/p/-dear-danny-i-found-my-boyfriends?s=r', | |
24 | 'md5': '0a63eacec877a1171a62cfa69710fcea', | |
25 | 'info_dict': { | |
26 | 'id': '51045592', | |
27 | 'ext': 'mpga', | |
28 | 'title': "🎧 Dear Danny: I found my boyfriend's secret Twitter account", | |
29 | 'description': 'md5:a57f2439319e56e0af92dd0c95d75797', | |
30 | 'thumbnail': 'md5:daa40b6b79249417c14ff8103db29639', | |
31 | 'uploader': 'Maybe Baby', | |
32 | 'uploader_id': '33628', | |
add96eb9 | 33 | }, |
612e31f5 E |
34 | }, { |
35 | 'url': 'https://andrewzimmern.substack.com/p/mussels-with-black-bean-sauce-recipe', | |
36 | 'md5': 'fd3c07077b02444ff0130715b5f632bb', | |
37 | 'info_dict': { | |
38 | 'id': '47368578', | |
39 | 'ext': 'mp4', | |
40 | 'title': 'Mussels with Black Bean Sauce: Recipe of the Week #7', | |
41 | 'description': 'md5:b96234a2906c7d854d5229818d889515', | |
42 | 'thumbnail': 'md5:e30bfaa9da40e82aa62354263a9dd232', | |
43 | 'uploader': "Andrew Zimmern's Spilled Milk ", | |
44 | 'uploader_id': '577659', | |
add96eb9 | 45 | }, |
612e31f5 E |
46 | }] |
47 | ||
48 | @classmethod | |
bfd973ec | 49 | def _extract_embed_urls(cls, url, webpage): |
612e31f5 E |
50 | if not re.search(r'<script[^>]+src=["\']https://substackcdn.com/[^"\']+\.js', webpage): |
51 | return | |
52 | ||
fbcc299b | 53 | mobj = re.search(r'{[^}]*\\?["\']subdomain\\?["\']\s*:\s*\\?["\'](?P<subdomain>[^\\"\']+)', webpage) |
612e31f5 E |
54 | if mobj: |
55 | parsed = urllib.parse.urlparse(url) | |
bfd973ec | 56 | yield parsed._replace(netloc=f'{mobj.group("subdomain")}.substack.com').geturl() |
add96eb9 | 57 | raise cls.StopExtraction |
612e31f5 | 58 | |
2f2dda3a | 59 | def _extract_video_formats(self, video_id, url): |
612e31f5 E |
60 | formats, subtitles = [], {} |
61 | for video_format in ('hls', 'mp4'): | |
2f2dda3a | 62 | video_url = urllib.parse.urljoin(url, f'/api/v1/video/upload/{video_id}/src?type={video_format}') |
612e31f5 E |
63 | |
64 | if video_format == 'hls': | |
65 | fmts, subs = self._extract_m3u8_formats_and_subtitles(video_url, video_id, 'mp4', fatal=False) | |
66 | formats.extend(fmts) | |
67 | self._merge_subtitles(subs, target=subtitles) | |
68 | else: | |
69 | formats.append({ | |
70 | 'url': video_url, | |
71 | 'ext': video_format, | |
72 | }) | |
73 | ||
74 | return formats, subtitles | |
75 | ||
76 | def _real_extract(self, url): | |
77 | display_id, username = self._match_valid_url(url).group('id', 'username') | |
78 | webpage = self._download_webpage(url, display_id) | |
79 | ||
12037d8b | 80 | webpage_info = self._parse_json(self._search_json( |
81 | r'window\._preloads\s*=\s*JSON\.parse\(', webpage, 'json string', | |
82 | display_id, transform_source=js_to_json, contains_pattern=r'"{(?s:.+)}"'), display_id) | |
612e31f5 | 83 | |
2f2dda3a UG |
84 | canonical_url = url |
85 | domain = traverse_obj(webpage_info, ('domainInfo', 'customDomain', {str})) | |
86 | if domain: | |
87 | canonical_url = urllib.parse.urlparse(url)._replace(netloc=domain).geturl() | |
88 | ||
612e31f5 E |
89 | post_type = webpage_info['post']['type'] |
90 | formats, subtitles = [], {} | |
91 | if post_type == 'podcast': | |
92 | formats, subtitles = [{'url': webpage_info['post']['podcast_url']}], {} | |
93 | elif post_type == 'video': | |
2f2dda3a | 94 | formats, subtitles = self._extract_video_formats(webpage_info['post']['videoUpload']['id'], canonical_url) |
612e31f5 E |
95 | else: |
96 | self.raise_no_formats(f'Page type "{post_type}" is not supported') | |
97 | ||
612e31f5 E |
98 | return { |
99 | 'id': str(webpage_info['post']['id']), | |
100 | 'formats': formats, | |
101 | 'subtitles': subtitles, | |
102 | 'title': traverse_obj(webpage_info, ('post', 'title')), | |
103 | 'description': traverse_obj(webpage_info, ('post', 'description')), | |
104 | 'thumbnail': traverse_obj(webpage_info, ('post', 'cover_image')), | |
105 | 'uploader': traverse_obj(webpage_info, ('pub', 'name')), | |
106 | 'uploader_id': str_or_none(traverse_obj(webpage_info, ('post', 'publication_id'))), | |
2f2dda3a | 107 | 'webpage_url': canonical_url, |
612e31f5 | 108 | } |